[WIP] Add FilterCompiler to SPI

.travis.yml

@@ -17,6 +17,7 @@ env:
     - TEST_SPECIFIC_MODULES=presto-accumulo
     - TEST_SPECIFIC_MODULES=presto-cassandra
     - TEST_SPECIFIC_MODULES=presto-hive
+    - TEST_SPECIFIC_MODULES=presto-hive TEST_FLAGS="-P test-hive-materialized"
     - TEST_SPECIFIC_MODULES=presto-main
     - TEST_SPECIFIC_MODULES=presto-mongodb
     - TEST_OTHER_MODULES=!presto-tests,!presto-raptor,!presto-accumulo,!presto-cassandra,!presto-hive,!presto-kudu,!presto-docs,!presto-server,!presto-server-rpm,!presto-main,!presto-mongodb

pom.xml

@@ -57,6 +57,7 @@
         <dep.assertj-core.version>3.8.0</dep.assertj-core.version>
         <dep.logback.version>1.2.3</dep.logback.version>
         <dep.parquet.version>1.10.0</dep.parquet.version>
+        <dep.nexus-staging-plugin.version>1.6.8</dep.nexus-staging-plugin.version>
 
         <!--
           America/Bahia_Banderas has:
@@ -1197,6 +1198,17 @@
                         <preparationGoals>clean verify -DskipTests</preparationGoals>
                     </configuration>
                 </plugin>
+
+                <plugin>
+                    <groupId>org.sonatype.plugins</groupId>
+                    <artifactId>nexus-staging-maven-plugin</artifactId>
+                    <version>${dep.nexus-staging-plugin.version}</version>
+                    <extensions>true</extensions>
+                    <configuration>
+                        <serverId>ossrh</serverId>
+                        <nexusUrl>https://oss.sonatype.org/</nexusUrl>
+                    </configuration>
+                </plugin>
             </plugins>
         </pluginManagement>
 
@@ -1208,13 +1220,6 @@
                 <extensions>true</extensions>
             </plugin>
 
-            <plugin>
-                <groupId>io.takari.maven.plugins</groupId>
-                <artifactId>provisio-maven-plugin</artifactId>
-                <version>0.1.40</version>
-                <extensions>true</extensions>
-            </plugin>
-
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-compiler-plugin</artifactId>
@@ -1281,5 +1286,16 @@
                 </plugins>
             </build>
         </profile>
+        <profile>
+            <id>deploy-to-ossrh</id>
+            <build>
+                <plugins>
+                    <plugin>
+                        <groupId>org.sonatype.plugins</groupId>
+                        <artifactId>nexus-staging-maven-plugin</artifactId>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
     </profiles>
 </project>

presto-array/src/main/java/com/facebook/presto/array/ReferenceCountMap.java

@@ -13,6 +13,7 @@
  */
 package com.facebook.presto.array;
 
+import com.facebook.presto.spi.block.AbstractMapBlock;
 import com.facebook.presto.spi.block.Block;
 import io.airlift.slice.SizeOf;
 import io.airlift.slice.Slice;
@@ -90,6 +91,9 @@ else if (key instanceof Slice) {
         else if (key.getClass().isArray()) {
             extraIdentity = getLength(key);
         }
+        else if (key instanceof AbstractMapBlock.HashTables) {
+            extraIdentity = (int) ((AbstractMapBlock.HashTables) key).getRetainedSizeInBytes();
+        }
         else {
             throw new IllegalArgumentException(format("Unsupported type for %s", key));
         }

presto-benchmark/src/main/java/com/facebook/presto/benchmark/CountAggregationBenchmark.java

@@ -19,7 +19,6 @@
 import com.facebook.presto.operator.aggregation.InternalAggregationFunction;
 import com.facebook.presto.sql.planner.plan.AggregationNode.Step;
 import com.facebook.presto.sql.planner.plan.PlanNodeId;
-import com.facebook.presto.sql.tree.QualifiedName;
 import com.facebook.presto.testing.LocalQueryRunner;
 import com.google.common.collect.ImmutableList;
 
@@ -44,7 +43,7 @@ protected List<? extends OperatorFactory> createOperatorFactories()
         OperatorFactory tableScanOperator = createTableScanOperator(0, new PlanNodeId("test"), "orders", "orderkey");
         FunctionManager functionManager = localQueryRunner.getMetadata().getFunctionManager();
         InternalAggregationFunction countFunction = functionManager.getAggregateFunctionImplementation(
-                functionManager.lookupFunction(QualifiedName.of("count"), fromTypes(BIGINT)));
+                functionManager.lookupFunction("count", fromTypes(BIGINT)));
         AggregationOperatorFactory aggregationOperator = new AggregationOperatorFactory(1, new PlanNodeId("test"), Step.SINGLE, ImmutableList.of(countFunction.bind(ImmutableList.of(0), Optional.empty())), false);
         return ImmutableList.of(tableScanOperator, aggregationOperator);
     }

presto-benchmark/src/main/java/com/facebook/presto/benchmark/DoubleSumAggregationBenchmark.java

@@ -20,7 +20,6 @@
 import com.facebook.presto.operator.aggregation.InternalAggregationFunction;
 import com.facebook.presto.sql.planner.plan.AggregationNode.Step;
 import com.facebook.presto.sql.planner.plan.PlanNodeId;
-import com.facebook.presto.sql.tree.QualifiedName;
 import com.facebook.presto.testing.LocalQueryRunner;
 import com.google.common.collect.ImmutableList;
 
@@ -45,7 +44,7 @@ protected List<? extends OperatorFactory> createOperatorFactories()
         OperatorFactory tableScanOperator = createTableScanOperator(0, new PlanNodeId("test"), "orders", "totalprice");
         FunctionManager functionManager = MetadataManager.createTestMetadataManager().getFunctionManager();
         InternalAggregationFunction doubleSum = functionManager.getAggregateFunctionImplementation(
-                functionManager.lookupFunction(QualifiedName.of("sum"), fromTypes(DOUBLE)));
+                functionManager.lookupFunction("sum", fromTypes(DOUBLE)));
         AggregationOperatorFactory aggregationOperator = new AggregationOperatorFactory(1, new PlanNodeId("test"), Step.SINGLE, ImmutableList.of(doubleSum.bind(ImmutableList.of(0), Optional.empty())), false);
         return ImmutableList.of(tableScanOperator, aggregationOperator);
     }

presto-benchmark/src/main/java/com/facebook/presto/benchmark/HandTpchQuery1.java

@@ -27,7 +27,6 @@
 import com.facebook.presto.spi.type.Type;
 import com.facebook.presto.sql.planner.plan.AggregationNode.Step;
 import com.facebook.presto.sql.planner.plan.PlanNodeId;
-import com.facebook.presto.sql.tree.QualifiedName;
 import com.facebook.presto.testing.LocalQueryRunner;
 import com.facebook.presto.util.DateTimeUtils;
 import com.google.common.collect.ImmutableList;
@@ -61,13 +60,13 @@ public HandTpchQuery1(LocalQueryRunner localQueryRunner)
 
         FunctionManager functionManager = localQueryRunner.getMetadata().getFunctionManager();
         longAverage = functionManager.getAggregateFunctionImplementation(
-                functionManager.lookupFunction(QualifiedName.of("avg"), fromTypes(BIGINT)));
+                functionManager.lookupFunction("avg", fromTypes(BIGINT)));
         doubleAverage = functionManager.getAggregateFunctionImplementation(
-                functionManager.lookupFunction(QualifiedName.of("avg"), fromTypes(DOUBLE)));
+                functionManager.lookupFunction("avg", fromTypes(DOUBLE)));
         doubleSum = functionManager.getAggregateFunctionImplementation(
-                functionManager.lookupFunction(QualifiedName.of("sum"), fromTypes(DOUBLE)));
+                functionManager.lookupFunction("sum", fromTypes(DOUBLE)));
         countFunction = functionManager.getAggregateFunctionImplementation(
-                functionManager.lookupFunction(QualifiedName.of("count"), ImmutableList.of()));
+                functionManager.lookupFunction("count", ImmutableList.of()));
     }
 
     @Override

presto-benchmark/src/main/java/com/facebook/presto/benchmark/HandTpchQuery6.java

@@ -29,7 +29,6 @@
 import com.facebook.presto.sql.gen.PageFunctionCompiler;
 import com.facebook.presto.sql.planner.plan.AggregationNode.Step;
 import com.facebook.presto.sql.planner.plan.PlanNodeId;
-import com.facebook.presto.sql.tree.QualifiedName;
 import com.facebook.presto.testing.LocalQueryRunner;
 import com.facebook.presto.util.DateTimeUtils;
 import com.google.common.collect.ImmutableList;
@@ -57,7 +56,7 @@ public HandTpchQuery6(LocalQueryRunner localQueryRunner)
         super(localQueryRunner, "hand_tpch_query_6", 10, 100);
         FunctionManager functionManager = localQueryRunner.getMetadata().getFunctionManager();
         doubleSum = functionManager.getAggregateFunctionImplementation(
-                functionManager.lookupFunction(QualifiedName.of("sum"), fromTypes(DOUBLE)));
+                functionManager.lookupFunction("sum", fromTypes(DOUBLE)));
     }
 
     @Override

presto-benchmark/src/main/java/com/facebook/presto/benchmark/HashAggregationBenchmark.java

@@ -20,7 +20,6 @@
 import com.facebook.presto.spi.type.Type;
 import com.facebook.presto.sql.planner.plan.AggregationNode.Step;
 import com.facebook.presto.sql.planner.plan.PlanNodeId;
-import com.facebook.presto.sql.tree.QualifiedName;
 import com.facebook.presto.testing.LocalQueryRunner;
 import com.google.common.collect.ImmutableList;
 import com.google.common.primitives.Ints;
@@ -45,7 +44,7 @@ public HashAggregationBenchmark(LocalQueryRunner localQueryRunner)
 
         FunctionManager functionManager = localQueryRunner.getMetadata().getFunctionManager();
         doubleSum = functionManager.getAggregateFunctionImplementation(
-                functionManager.lookupFunction(QualifiedName.of("sum"), fromTypes(DOUBLE)));
+                functionManager.lookupFunction("sum", fromTypes(DOUBLE)));
     }
 
     @Override

presto-docs/src/main/sphinx/functions/string.rst

@@ -116,14 +116,16 @@ String Functions
 
     Splits ``string`` by ``entryDelimiter`` and ``keyValueDelimiter`` and returns a map.
     ``entryDelimiter`` splits ``string`` into key-value pairs. ``keyValueDelimiter`` splits
-    each pair into key and value.
+    each pair into key and value. Note that ``entryDelimiter`` and ``keyValueDelimiter`` are
+    interpreted literally, i.e., as full string matches.
 
 .. function:: split_to_map(string, entryDelimiter, keyValueDelimiter, function(k, v1, v2, res)) -> map<varchar, varchar>
 
     Splits ``string`` by ``entryDelimiter`` and ``keyValueDelimiter`` and returns a map.
     ``entryDelimiter`` splits ``string`` into key-value pairs. ``keyValueDelimiter`` splits
-    each pair into key and value. ``function(k, v1, v2, res)`` is invoked in cases of duplicate
-    keys to resolve the value that should be in the map.
+    each pair into key and value. Note that ``entryDelimiter`` and ``keyValueDelimiter`` are
+    interpreted literally, i.e., as full string matches. ``function(k, v1, v2, res)``
+    is invoked in cases of duplicate keys to resolve the value that should be in the map.
 
         SELECT(split_to_map('a:1;b:2;a:3', ';', ':', (k, v1, v2) -> v1)); -- {"a": "1", "b": "2"}
         SELECT(split_to_map('a:1;b:2;a:3', ';', ':', (k, v1, v2) -> CONCAT(v1, v2))); -- {"a": "13", "b": "2"}
@@ -134,6 +136,8 @@ String Functions
     containing an array of values for each unique key. ``entryDelimiter`` splits ``string``
     into key-value pairs. ``keyValueDelimiter`` splits each pair into key and value. The
     values for each key will be in the same order as they appeared in ``string``.
+    Note that ``entryDelimiter`` and ``keyValueDelimiter`` are interpreted literally,
+    i.e., as full string matches.
 
 .. function:: strpos(string, substring) -> bigint
 

presto-docs/src/main/sphinx/release.rst

@@ -5,6 +5,7 @@ Release Notes
 .. toctree::
     :maxdepth: 1
 
+    release/release-0.219
     release/release-0.218
     release/release-0.217
     release/release-0.216

presto-docs/src/main/sphinx/release/release-0.219.rst

@@ -0,0 +1,68 @@
+=============
+Release 0.219
+=============
+
+General Changes
+---------------
+
+* Fix a correctness bug for queries with a ``LIMIT`` on top of a ``FULL OUTER JOIN``.
+* Fix a correctness bug in how word boundaries are handled for regular expression functions when using the Joni regex engine.
+* Fix query failures caused by Parquet predicate pushdown for ``SMALLINT`` and ``TINYINT`` types.
+* Fix query failures when view columns were fully qualified with catalog and/or schema.
+* Fix query failures when creating and reading an unpartitioned Hive table within the same transaction.
+* Fix an issue that would disable a performance optimization if two tables with mismatching
+  but compatible bucket counts are joined and one of the tables has a ``$bucket`` filter.
+* Fix an issue that may cause procedure calls to fail if no queries were run after the server started up.
+* Fix an issue where the properties supported by the ``ANALYZE`` statement for a given connector would remain in the ``system.metadata.analyze_properties`` table
+  even after the connector was removed.
+* Add :func:`ST_Length` for ``SphericalGeography`` type..
+* Add ``view_owner`` column to the ``information_schema.views`` system table.
+* Add a ``JSON`` version of the query plan to ``QueryCompletedEvent``.
+* Add support for creating warnings during parsing.
+* Add a warning for using the ``current_role`` reserved word as an identifier.
+* Add support for using the empty string as a delimiter for the :func:`split` function.
+  When an empty string is used as the delimiter, the string will be split into individual characters.
+
+Raptor Changes
+--------------
+* Add ``raptor.minimum-node-count`` configuration property. If the number of nodes in a cluster is less than the configured value,
+  data recovery and reassignment will not run.
+* Add ``raptor.startup-grace-period`` configuration property to specify the delay for the initial bucket balancer run after the coordinator startup.
+
+Hive Connector Changes
+----------------------
+
+* Add ``hive.ignore-table-bucketing`` configuration property and ``ignore_table_bucketing`` session property.
+  When set to true, these properties enable reading from partitions whose bucketing scheme does not match the table bucketing scheme.
+
+Verifier Changes
+----------------
+* Fix an issue where the determinism check for ``INSERT`` queries were not run when there was a result mismatch.
+* Fix an issue where the checksum query was treated as the main query, and the main query was treated as part of the setup queries
+  for ``INSERT`` query verification.
+* Fix query timeout enforcement by replacing local timer with the ``query_max_execution_time`` session property.
+* Improve result comparison for floating point columns by using relative errors.
+  Replace configuration property ``expected-double-precision`` with ``relative-error-margin``.
+* Improve result comparison for orderable array columns by applying :func:`array_sort` before :func:`checksum`.
+* Improve result comparison for ``SELECT`` queries by rewriting ``SELECT`` queries as ``CREATE TABLE AS`` and using checksum queries to verify the results.
+  This eliminates the row count limit for ``SELECT`` queries.
+* Reuse initial control query results for the determinism check. This reduces the maximum number of control query runs and eliminates the test query reruns.
+* Add support for retrying transient query failures using configuration properties ``presto.max-attempts``, ``presto.min-backoff-delay``,
+  ``presto.max-backoff-delay``, ``presto.backoff-scale-factor``. Similar configurations prefixed with ``cluster-connection`` instead of ``presto``
+  are introduced for retrying transient network failures when communicating with the coordinator. Intermediate failures are recorded and emitted in the output.
+* Add support for automatically resolving certain kinds of failures including exceeding the global memory limit and exceeding time limit.
+* Add configuration properties ``metadata-timeout`` and ``checksum-timeout`` to set the timeouts for metadata queries
+  (i.e., describe queries that read table schema) and checksum queries.
+* Add ``source-query.table-name`` configuration property to specify the name of the MySQL table from which the verifier queries will loaded.
+* Add configuration property ``human-readable.log-file`` to allow human-readable verification results to be logged into the specified file instead of ``stdout``.
+* Rename configuration properties ``query-database`` to ``source-query.database``, ``suites`` to ``source-query.suites``,
+  ``max_queries`` to ``source-query.max-queries-per-suite``, ``event-client`` to ``event-clients``, ``event-log-file`` to ``json.log-file``,
+  ``run-id`` to ``test-id``, ``thread-count`` to ``max-concurrency``, ``control-gateway`` to ``control.jdbc_url``, ``test-gateway`` to ``test.jdbc_url``,
+  ``shadow-writes.control-table-prefix`` to ``control.table-prefix``, and ``shadow-writes.test-table-prefix`` to ``test.table-prefix``.
+* Remove configuration properties ``control.query-types``, ``test.query-types``, ``source``, ``max-row-count``, ``always-report``,
+  ``skip-correctness-regex``, ``check-correctness``, ``skip-cpu-check-regex``, ``check-cpu``, ``explain-only``, ``verbose-results-comparison``,
+  ``quiet``, ``control-teardown-retries``, ``test-teardown-retries``, and ``shadow-writes``.
+
+SPI Changes
+-----------
+* Expose ``RowExpression`` to SPI, to allow passing resolved expressions to connectors.

presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestSpatialPartitioningInternalAggregation.java

@@ -29,7 +29,6 @@
 import com.facebook.presto.spi.Page;
 import com.facebook.presto.spi.block.Block;
 import com.facebook.presto.spi.block.BlockBuilder;
-import com.facebook.presto.sql.tree.QualifiedName;
 import com.google.common.collect.ImmutableList;
 import com.google.common.primitives.Ints;
 import org.testng.annotations.BeforeClass;
@@ -98,7 +97,7 @@ private InternalAggregationFunction getFunction()
     {
         FunctionManager functionManager = functionAssertions.getMetadata().getFunctionManager();
         return functionManager.getAggregateFunctionImplementation(
-                functionManager.lookupFunction(QualifiedName.of("spatial_partitioning"), fromTypes(GEOMETRY, INTEGER)));
+                functionManager.lookupFunction("spatial_partitioning", fromTypes(GEOMETRY, INTEGER)));
     }
 
     private List<OGCGeometry> makeGeometries()

presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/aggregation/AbstractTestGeoAggregationFunctions.java

@@ -22,7 +22,6 @@
 import com.facebook.presto.plugin.geospatial.GeoPlugin;
 import com.facebook.presto.spi.Page;
 import com.facebook.presto.spi.type.Type;
-import com.facebook.presto.sql.tree.QualifiedName;
 import io.airlift.slice.Slice;
 import org.testng.annotations.BeforeClass;
 
@@ -52,7 +51,7 @@ public void registerFunctions()
         functionAssertions.getMetadata().addFunctions(extractFunctions(plugin.getFunctions()));
         FunctionManager functionManager = functionAssertions.getMetadata().getFunctionManager();
         function = functionManager.getAggregateFunctionImplementation(
-                functionManager.lookupFunction(QualifiedName.of(getFunctionName()), fromTypes(GEOMETRY)));
+                functionManager.lookupFunction(getFunctionName(), fromTypes(GEOMETRY)));
     }
 
     protected void assertAggregatedGeometries(String testDescription, String expectedWkt, String... wkts)