requiredConfig, ConnectorContext context)
{
requireNonNull(requiredConfig, "requiredConfig is null");
try {
// A plugin is not required to use Guice; it is just very convenient
Bootstrap app = new Bootstrap(
new JsonModule(),
- new ExampleModule(connectorId, context.getTypeManager()));
+ new ExampleModule(catalogName, context.getTypeManager()));
Injector injector = app
.strictConfig()
diff --git a/presto-example-http/src/test/java/com/facebook/presto/example/TestExampleMetadata.java b/presto-example-http/src/test/java/com/facebook/presto/example/TestExampleMetadata.java
index 09bb1ffeeaf80..4ca7e73431162 100644
--- a/presto-example-http/src/test/java/com/facebook/presto/example/TestExampleMetadata.java
+++ b/presto-example-http/src/test/java/com/facebook/presto/example/TestExampleMetadata.java
@@ -144,9 +144,11 @@ public void getColumnMetadata()
@Test(expectedExceptions = PrestoException.class)
public void testCreateTable()
{
- metadata.createTable(SESSION, new ConnectorTableMetadata(
- new SchemaTableName("example", "foo"),
- ImmutableList.of(new ColumnMetadata("text", createUnboundedVarcharType()))),
+ metadata.createTable(
+ SESSION,
+ new ConnectorTableMetadata(
+ new SchemaTableName("example", "foo"),
+ ImmutableList.of(new ColumnMetadata("text", createUnboundedVarcharType()))),
false);
}
diff --git a/presto-geospatial-toolkit/pom.xml b/presto-geospatial-toolkit/pom.xml
index 62cd09e613239..8345fc1a5c7a9 100644
--- a/presto-geospatial-toolkit/pom.xml
+++ b/presto-geospatial-toolkit/pom.xml
@@ -4,7 +4,7 @@
com.facebook.presto
presto-root
- 0.209-SNAPSHOT
+ 0.216-SNAPSHOT
presto-geospatial-toolkit
@@ -15,6 +15,11 @@
+
+ org.openjdk.jol
+ jol-core
+
+
com.esri.geometry
esri-geometry-api
@@ -25,6 +30,11 @@
jts-core
+
+ com.fasterxml.jackson.core
+ jackson-annotations
+
+
com.google.guava
guava
@@ -35,6 +45,11 @@
slice
+
+ io.airlift
+ json
+
+
com.google.code.findbugs
jsr305
@@ -58,5 +73,6 @@
testng
test
+
diff --git a/presto-geospatial-toolkit/src/main/java/com/facebook/presto/geospatial/GeometryUtils.java b/presto-geospatial-toolkit/src/main/java/com/facebook/presto/geospatial/GeometryUtils.java
index a48f06c0ce7ea..066e1dbbcef7b 100644
--- a/presto-geospatial-toolkit/src/main/java/com/facebook/presto/geospatial/GeometryUtils.java
+++ b/presto-geospatial-toolkit/src/main/java/com/facebook/presto/geospatial/GeometryUtils.java
@@ -33,7 +33,7 @@ private GeometryUtils() {}
/**
* Copy of com.esri.core.geometry.Interop.translateFromAVNaN
- *
+ *
* deserializeEnvelope needs to recognize custom NAN values generated by
* ESRI's serialization of empty geometries.
*/
@@ -44,7 +44,7 @@ private static double translateFromAVNaN(double n)
/**
* Copy of com.esri.core.geometry.Interop.translateToAVNaN
- *
+ *
* JtsGeometrySerde#serialize must serialize NaN's the same way ESRI library does to achieve binary compatibility
*/
public static double translateToAVNaN(double n)
diff --git a/presto-geospatial-toolkit/src/main/java/com/facebook/presto/geospatial/KdbTree.java b/presto-geospatial-toolkit/src/main/java/com/facebook/presto/geospatial/KdbTree.java
new file mode 100644
index 0000000000000..6b15cae5196f9
--- /dev/null
+++ b/presto-geospatial-toolkit/src/main/java/com/facebook/presto/geospatial/KdbTree.java
@@ -0,0 +1,363 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.facebook.presto.geospatial;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Predicate;
+import com.google.common.collect.ComparisonChain;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.OptionalInt;
+
+import static com.facebook.presto.geospatial.KdbTree.Node.newInternal;
+import static com.facebook.presto.geospatial.KdbTree.Node.newLeaf;
+import static com.google.common.base.Preconditions.checkArgument;
+import static java.util.Objects.requireNonNull;
+
+/**
+ * 2-dimensional K-D-B Tree
+ * see https://en.wikipedia.org/wiki/K-D-B-tree
+ */
+public class KdbTree
+{
+ private static final int MAX_LEVELS = 10_000;
+
+ private final Node root;
+
+ public static final class Node
+ {
+ private final Rectangle extent;
+ private final OptionalInt leafId;
+ private final Optional left;
+ private final Optional right;
+
+ public static Node newLeaf(Rectangle extent, int leafId)
+ {
+ return new Node(extent, OptionalInt.of(leafId), Optional.empty(), Optional.empty());
+ }
+
+ public static Node newInternal(Rectangle extent, Node left, Node right)
+ {
+ return new Node(extent, OptionalInt.empty(), Optional.of(left), Optional.of(right));
+ }
+
+ @JsonCreator
+ public Node(
+ @JsonProperty("extent") Rectangle extent,
+ @JsonProperty("leafId") OptionalInt leafId,
+ @JsonProperty("left") Optional left,
+ @JsonProperty("right") Optional right)
+ {
+ this.extent = requireNonNull(extent, "extent is null");
+ this.leafId = requireNonNull(leafId, "leafId is null");
+ this.left = requireNonNull(left, "left is null");
+ this.right = requireNonNull(right, "right is null");
+ if (leafId.isPresent()) {
+ checkArgument(leafId.getAsInt() >= 0, "leafId must be >= 0");
+ checkArgument(!left.isPresent(), "Leaf node cannot have left child");
+ checkArgument(!right.isPresent(), "Leaf node cannot have right child");
+ }
+ else {
+ checkArgument(left.isPresent(), "Intermediate node must have left child");
+ checkArgument(right.isPresent(), "Intermediate node must have right child");
+ }
+ }
+
+ @JsonProperty
+ public Rectangle getExtent()
+ {
+ return extent;
+ }
+
+ @JsonProperty
+ public OptionalInt getLeafId()
+ {
+ return leafId;
+ }
+
+ @JsonProperty
+ public Optional getLeft()
+ {
+ return left;
+ }
+
+ @JsonProperty
+ public Optional getRight()
+ {
+ return right;
+ }
+
+ @Override
+ public boolean equals(Object obj)
+ {
+ if (obj == null) {
+ return false;
+ }
+
+ if (!(obj instanceof Node)) {
+ return false;
+ }
+
+ Node other = (Node) obj;
+ return this.extent.equals(other.extent)
+ && Objects.equals(this.leafId, other.leafId)
+ && Objects.equals(this.left, other.left)
+ && Objects.equals(this.right, other.right);
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects.hash(extent, leafId, left, right);
+ }
+ }
+
+ @JsonCreator
+ public KdbTree(@JsonProperty("root") Node root)
+ {
+ this.root = requireNonNull(root, "root is null");
+ }
+
+ @JsonProperty
+ public Node getRoot()
+ {
+ return root;
+ }
+
+ @Override
+ public boolean equals(Object obj)
+ {
+ if (obj == null) {
+ return false;
+ }
+
+ if (!(obj instanceof KdbTree)) {
+ return false;
+ }
+
+ KdbTree other = (KdbTree) obj;
+ return this.root.equals(other.root);
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects.hash(root);
+ }
+
+ public Map getLeaves()
+ {
+ ImmutableMap.Builder leaves = ImmutableMap.builder();
+ addLeaves(root, leaves, node -> true);
+ return leaves.build();
+ }
+
+ public Map findIntersectingLeaves(Rectangle envelope)
+ {
+ ImmutableMap.Builder leaves = ImmutableMap.builder();
+ addLeaves(root, leaves, node -> node.extent.intersects(envelope));
+ return leaves.build();
+ }
+
+ private static void addLeaves(Node node, ImmutableMap.Builder leaves, Predicate predicate)
+ {
+ if (!predicate.apply(node)) {
+ return;
+ }
+
+ if (node.leafId.isPresent()) {
+ leaves.put(node.leafId.getAsInt(), node.extent);
+ }
+ else {
+ addLeaves(node.left.get(), leaves, predicate);
+ addLeaves(node.right.get(), leaves, predicate);
+ }
+ }
+
+ private interface SplitDimension
+ {
+ Comparator getComparator();
+
+ double getValue(Rectangle rectangle);
+
+ SplitResult split(Rectangle rectangle, double value);
+ }
+
+ private static final SplitDimension BY_X = new SplitDimension() {
+ private final Comparator comparator = (first, second) -> ComparisonChain.start()
+ .compare(first.getXMin(), second.getXMin())
+ .compare(first.getYMin(), second.getYMin())
+ .result();
+
+ @Override
+ public Comparator getComparator()
+ {
+ return comparator;
+ }
+
+ @Override
+ public double getValue(Rectangle rectangle)
+ {
+ return rectangle.getXMin();
+ }
+
+ @Override
+ public SplitResult split(Rectangle rectangle, double x)
+ {
+ checkArgument(rectangle.getXMin() < x && x < rectangle.getXMax());
+ return new SplitResult<>(
+ new Rectangle(rectangle.getXMin(), rectangle.getYMin(), x, rectangle.getYMax()),
+ new Rectangle(x, rectangle.getYMin(), rectangle.getXMax(), rectangle.getYMax()));
+ }
+ };
+
+ private static final SplitDimension BY_Y = new SplitDimension() {
+ private final Comparator comparator = (first, second) -> ComparisonChain.start()
+ .compare(first.getYMin(), second.getYMin())
+ .compare(first.getXMin(), second.getXMin())
+ .result();
+
+ @Override
+ public Comparator getComparator()
+ {
+ return comparator;
+ }
+
+ @Override
+ public double getValue(Rectangle rectangle)
+ {
+ return rectangle.getYMin();
+ }
+
+ @Override
+ public SplitResult split(Rectangle rectangle, double y)
+ {
+ checkArgument(rectangle.getYMin() < y && y < rectangle.getYMax());
+ return new SplitResult<>(
+ new Rectangle(rectangle.getXMin(), rectangle.getYMin(), rectangle.getXMax(), y),
+ new Rectangle(rectangle.getXMin(), y, rectangle.getXMax(), rectangle.getYMax()));
+ }
+ };
+
+ private static final class LeafIdAllocator
+ {
+ private int nextId;
+
+ public int next()
+ {
+ return nextId++;
+ }
+ }
+
+ public static KdbTree buildKdbTree(int maxItemsPerNode, Rectangle extent, List items)
+ {
+ checkArgument(maxItemsPerNode > 0, "maxItemsPerNode must be > 0");
+ requireNonNull(extent, "extent is null");
+ requireNonNull(items, "items is null");
+ return new KdbTree(buildKdbTreeNode(maxItemsPerNode, 0, extent, items, new LeafIdAllocator()));
+ }
+
+ private static Node buildKdbTreeNode(int maxItemsPerNode, int level, Rectangle extent, List items, LeafIdAllocator leafIdAllocator)
+ {
+ checkArgument(maxItemsPerNode > 0, "maxItemsPerNode must be > 0");
+ checkArgument(level >= 0, "level must be >= 0");
+ checkArgument(level <= MAX_LEVELS, "level must be <= 10,000");
+ requireNonNull(extent, "extent is null");
+ requireNonNull(items, "items is null");
+
+ if (items.size() <= maxItemsPerNode || level == MAX_LEVELS) {
+ return newLeaf(extent, leafIdAllocator.next());
+ }
+
+ // Split over longer side
+ boolean splitVertically = extent.getWidth() >= extent.getHeight();
+ Optional> splitResult = trySplit(splitVertically ? BY_X : BY_Y, maxItemsPerNode, level, extent, items, leafIdAllocator);
+ if (!splitResult.isPresent()) {
+ // Try spitting by the other side
+ splitResult = trySplit(splitVertically ? BY_Y : BY_X, maxItemsPerNode, level, extent, items, leafIdAllocator);
+ }
+
+ if (!splitResult.isPresent()) {
+ return newLeaf(extent, leafIdAllocator.next());
+ }
+
+ return newInternal(extent, splitResult.get().getLeft(), splitResult.get().getRight());
+ }
+
+ private static final class SplitResult
+ {
+ private final T left;
+ private final T right;
+
+ private SplitResult(T left, T right)
+ {
+ this.left = requireNonNull(left, "left is null");
+ this.right = requireNonNull(right, "right is null");
+ }
+
+ public T getLeft()
+ {
+ return left;
+ }
+
+ public T getRight()
+ {
+ return right;
+ }
+ }
+
+ private static Optional> trySplit(SplitDimension splitDimension, int maxItemsPerNode, int level, Rectangle extent, List items, LeafIdAllocator leafIdAllocator)
+ {
+ checkArgument(items.size() > 1, "Number of items to split must be > 1");
+
+ // Sort envelopes by xMin or yMin
+ List sortedItems = ImmutableList.sortedCopyOf(splitDimension.getComparator(), items);
+
+ // Find a mid-point
+ int middleIndex = (sortedItems.size() - 1) / 2;
+ Rectangle middleEnvelope = sortedItems.get(middleIndex);
+ double splitValue = splitDimension.getValue(middleEnvelope);
+ int splitIndex = middleIndex;
+
+ // skip over duplicate values
+ while (splitIndex < sortedItems.size() && splitDimension.getValue(sortedItems.get(splitIndex)) == splitValue) {
+ splitIndex++;
+ }
+
+ // all values between left-of-middle and the end are the same, so can't split
+ if (splitIndex == sortedItems.size()) {
+ return Optional.empty();
+ }
+
+ // about half of the objects are <= splitValue, the rest are >= next value
+ // assuming the input set of objects is a sample from a much larger set,
+ // let's split in the middle; this way objects from the larger set with values
+ // between splitValue and next value will get split somewhat evenly into left
+ // and right partitions
+ splitValue = (splitValue + splitDimension.getValue(sortedItems.get(splitIndex))) / 2;
+
+ SplitResult childExtents = splitDimension.split(extent, splitValue);
+
+ return Optional.of(new SplitResult(
+ buildKdbTreeNode(maxItemsPerNode, level + 1, childExtents.getLeft(), sortedItems.subList(0, splitIndex), leafIdAllocator),
+ buildKdbTreeNode(maxItemsPerNode, level + 1, childExtents.getRight(), sortedItems.subList(splitIndex, sortedItems.size()), leafIdAllocator)));
+ }
+}
diff --git a/presto-geospatial-toolkit/src/main/java/com/facebook/presto/geospatial/KdbTreeUtils.java b/presto-geospatial-toolkit/src/main/java/com/facebook/presto/geospatial/KdbTreeUtils.java
new file mode 100644
index 0000000000000..a2899bf4571eb
--- /dev/null
+++ b/presto-geospatial-toolkit/src/main/java/com/facebook/presto/geospatial/KdbTreeUtils.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.facebook.presto.geospatial;
+
+import io.airlift.json.JsonCodec;
+import io.airlift.json.JsonCodecFactory;
+
+import static java.util.Objects.requireNonNull;
+
+public class KdbTreeUtils
+{
+ private static final JsonCodec KDB_TREE_CODEC = new JsonCodecFactory().jsonCodec(KdbTree.class);
+
+ private KdbTreeUtils() {}
+
+ public static KdbTree fromJson(String json)
+ {
+ requireNonNull(json, "json is null");
+ return KDB_TREE_CODEC.fromJson(json);
+ }
+
+ public static String toJson(KdbTree kdbTree)
+ {
+ requireNonNull(kdbTree, "kdbTree is null");
+ return KDB_TREE_CODEC.toJson(kdbTree);
+ }
+}
diff --git a/presto-geospatial-toolkit/src/main/java/com/facebook/presto/geospatial/Rectangle.java b/presto-geospatial-toolkit/src/main/java/com/facebook/presto/geospatial/Rectangle.java
new file mode 100644
index 0000000000000..8f7794f2c53cd
--- /dev/null
+++ b/presto-geospatial-toolkit/src/main/java/com/facebook/presto/geospatial/Rectangle.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.facebook.presto.geospatial;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.openjdk.jol.info.ClassLayout;
+
+import java.util.Objects;
+
+import static com.google.common.base.MoreObjects.toStringHelper;
+import static com.google.common.base.Preconditions.checkArgument;
+import static java.lang.Math.max;
+import static java.lang.Math.min;
+import static java.util.Objects.requireNonNull;
+
+public final class Rectangle
+{
+ private static final int INSTANCE_SIZE = ClassLayout.parseClass(Rectangle.class).instanceSize();
+
+ private final double xMin;
+ private final double yMin;
+ private final double xMax;
+ private final double yMax;
+
+ @JsonCreator
+ public Rectangle(
+ @JsonProperty("xmin") double xMin,
+ @JsonProperty("ymin") double yMin,
+ @JsonProperty("xmax") double xMax,
+ @JsonProperty("ymax") double yMax)
+ {
+ checkArgument(xMin <= xMax, "xMin is greater than xMax");
+ checkArgument(yMin <= yMax, "yMin is greater than yMax");
+ this.xMin = xMin;
+ this.yMin = yMin;
+ this.xMax = xMax;
+ this.yMax = yMax;
+ }
+
+ @JsonProperty
+ public double getXMin()
+ {
+ return xMin;
+ }
+
+ @JsonProperty
+ public double getYMin()
+ {
+ return yMin;
+ }
+
+ @JsonProperty
+ public double getXMax()
+ {
+ return xMax;
+ }
+
+ @JsonProperty
+ public double getYMax()
+ {
+ return yMax;
+ }
+
+ public double getWidth()
+ {
+ return xMax - xMin;
+ }
+ public double getHeight()
+ {
+ return yMax - yMin;
+ }
+
+ public boolean intersects(Rectangle other)
+ {
+ requireNonNull(other, "other is null");
+ return this.xMin <= other.xMax && this.xMax >= other.xMin && this.yMin <= other.yMax && this.yMax >= other.yMin;
+ }
+
+ public Rectangle intersection(Rectangle other)
+ {
+ requireNonNull(other, "other is null");
+ if (!intersects(other)) {
+ return null;
+ }
+
+ return new Rectangle(max(this.xMin, other.xMin), max(this.yMin, other.yMin), min(this.xMax, other.xMax), min(this.yMax, other.yMax));
+ }
+
+ public Rectangle merge(Rectangle other)
+ {
+ return new Rectangle(min(this.xMin, other.xMin), min(this.yMin, other.yMin), max(this.xMax, other.xMax), max(this.yMax, other.yMax));
+ }
+
+ public int estimateMemorySize()
+ {
+ return INSTANCE_SIZE;
+ }
+
+ @Override
+ public boolean equals(Object obj)
+ {
+ if (obj == null) {
+ return false;
+ }
+
+ if (!(obj instanceof Rectangle)) {
+ return false;
+ }
+
+ Rectangle other = (Rectangle) obj;
+ return other.xMin == this.xMin && other.yMin == this.yMin && other.xMax == this.xMax && other.yMax == this.yMax;
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects.hash(xMin, yMin, xMax, yMax);
+ }
+
+ @Override
+ public String toString()
+ {
+ return toStringHelper(this)
+ .add("xMin", xMin)
+ .add("yMin", yMin)
+ .add("xMax", xMax)
+ .add("yMax", yMax)
+ .toString();
+ }
+}
diff --git a/presto-geospatial-toolkit/src/main/java/com/facebook/presto/geospatial/serde/GeometrySerde.java b/presto-geospatial-toolkit/src/main/java/com/facebook/presto/geospatial/serde/GeometrySerde.java
index 04fe91805e53a..1f51a909acbd2 100644
--- a/presto-geospatial-toolkit/src/main/java/com/facebook/presto/geospatial/serde/GeometrySerde.java
+++ b/presto-geospatial-toolkit/src/main/java/com/facebook/presto/geospatial/serde/GeometrySerde.java
@@ -278,10 +278,7 @@ public static Envelope deserializeEnvelope(Slice shape)
{
requireNonNull(shape, "shape is null");
BasicSliceInput input = shape.getInput();
-
- if (input.available() == 0) {
- return null;
- }
+ verify(input.available() > 0);
int length = input.available() - 1;
GeometrySerializationType type = GeometrySerializationType.getForCode(input.readByte());
diff --git a/presto-geospatial-toolkit/src/test/java/com/facebook/presto/geospatial/TestKdbTree.java b/presto-geospatial-toolkit/src/test/java/com/facebook/presto/geospatial/TestKdbTree.java
new file mode 100644
index 0000000000000..16404fd1bb5ee
--- /dev/null
+++ b/presto-geospatial-toolkit/src/test/java/com/facebook/presto/geospatial/TestKdbTree.java
@@ -0,0 +1,283 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.facebook.presto.geospatial;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import org.testng.annotations.Test;
+
+import java.util.Map;
+import java.util.Set;
+
+import static com.facebook.presto.geospatial.KdbTree.buildKdbTree;
+import static org.testng.Assert.assertEquals;
+
+public class TestKdbTree
+{
+ @Test
+ public void testSerde()
+ {
+ Rectangle extent = new Rectangle(0, 0, 9, 4);
+ ImmutableList.Builder rectangles = ImmutableList.builder();
+ for (double x = 0; x < 10; x += 1) {
+ for (double y = 0; y < 5; y += 1) {
+ rectangles.add(new Rectangle(x, y, x + 0.1, y + 0.2));
+ }
+ }
+
+ testSerializationRoundtrip(buildKdbTree(100, extent, rectangles.build()));
+ testSerializationRoundtrip(buildKdbTree(20, extent, rectangles.build()));
+ testSerializationRoundtrip(buildKdbTree(10, extent, rectangles.build()));
+ }
+
+ private void testSerializationRoundtrip(KdbTree tree)
+ {
+ KdbTree treeCopy = KdbTreeUtils.fromJson(KdbTreeUtils.toJson(tree));
+ assertEquals(treeCopy, tree);
+ }
+
+ @Test
+ public void testSinglePartition()
+ {
+ testSinglePartition(0, 0);
+ testSinglePartition(1, 2);
+ }
+
+ private void testSinglePartition(double width, double height)
+ {
+ Rectangle extent = new Rectangle(0, 0, 9, 4);
+ ImmutableList.Builder rectangles = ImmutableList.builder();
+ for (double x = 0; x < 10; x += 1) {
+ for (double y = 0; y < 5; y += 1) {
+ rectangles.add(new Rectangle(x, y, x + width, y + height));
+ }
+ }
+
+ KdbTree tree = buildKdbTree(100, extent, rectangles.build());
+
+ assertEquals(tree.getLeaves().size(), 1);
+
+ Map.Entry entry = Iterables.getOnlyElement(tree.getLeaves().entrySet());
+ assertEquals(entry.getKey().intValue(), 0);
+ assertEquals(entry.getValue(), extent);
+ }
+
+ @Test
+ public void testSplitVertically()
+ {
+ testSplitVertically(0, 0);
+ testSplitVertically(1, 2);
+ }
+
+ private void testSplitVertically(double width, double height)
+ {
+ Rectangle extent = new Rectangle(0, 0, 9, 4);
+ ImmutableList.Builder rectangles = ImmutableList.builder();
+ for (int x = 0; x < 10; x++) {
+ for (int y = 0; y < 5; y++) {
+ rectangles.add(new Rectangle(x, y, x + width, y + height));
+ }
+ }
+
+ KdbTree treeCopy = buildKdbTree(25, extent, rectangles.build());
+
+ Map leafNodes = treeCopy.getLeaves();
+ assertEquals(leafNodes.size(), 2);
+ assertEquals(leafNodes.keySet(), ImmutableSet.of(0, 1));
+ assertEquals(leafNodes.get(0), new Rectangle(0, 0, 4.5, 4));
+ assertEquals(leafNodes.get(1), new Rectangle(4.5, 0, 9, 4));
+
+ assertPartitions(treeCopy, new Rectangle(1, 1, 2, 2), ImmutableSet.of(0));
+ assertPartitions(treeCopy, new Rectangle(1, 1, 5, 2), ImmutableSet.of(0, 1));
+ }
+
+ @Test
+ public void testSplitHorizontally()
+ {
+ testSplitHorizontally(0, 0);
+ testSplitHorizontally(1, 2);
+ }
+
+ private void testSplitHorizontally(double width, double height)
+ {
+ Rectangle extent = new Rectangle(0, 0, 4, 9);
+ ImmutableList.Builder rectangles = ImmutableList.builder();
+ for (int x = 0; x < 5; x++) {
+ for (int y = 0; y < 10; y++) {
+ rectangles.add(new Rectangle(x, y, x + width, y + height));
+ }
+ }
+
+ KdbTree tree = buildKdbTree(25, extent, rectangles.build());
+
+ Map leafNodes = tree.getLeaves();
+ assertEquals(leafNodes.size(), 2);
+ assertEquals(leafNodes.keySet(), ImmutableSet.of(0, 1));
+ assertEquals(leafNodes.get(0), new Rectangle(0, 0, 4, 4.5));
+ assertEquals(leafNodes.get(1), new Rectangle(0, 4.5, 4, 9));
+
+ // points inside and outside partitions
+ assertPartitions(tree, new Rectangle(1, 1, 1, 1), ImmutableSet.of(0));
+ assertPartitions(tree, new Rectangle(1, 6, 1, 6), ImmutableSet.of(1));
+ assertPartitions(tree, new Rectangle(5, 1, 5, 1), ImmutableSet.of());
+
+ // point on the border separating two partitions
+ assertPartitions(tree, new Rectangle(1, 4.5, 1, 4.5), ImmutableSet.of(0, 1));
+
+ // rectangles
+ assertPartitions(tree, new Rectangle(1, 1, 2, 2), ImmutableSet.of(0));
+ assertPartitions(tree, new Rectangle(1, 6, 2, 7), ImmutableSet.of(1));
+ assertPartitions(tree, new Rectangle(1, 1, 2, 5), ImmutableSet.of(0, 1));
+ assertPartitions(tree, new Rectangle(5, 1, 6, 2), ImmutableSet.of());
+ }
+
+ private void assertPartitions(KdbTree kdbTree, Rectangle envelope, Set partitions)
+ {
+ Map matchingNodes = kdbTree.findIntersectingLeaves(envelope);
+ assertEquals(matchingNodes.size(), partitions.size());
+ assertEquals(matchingNodes.keySet(), partitions);
+ }
+
+ @Test
+ public void testEvenDistribution()
+ {
+ testEvenDistribution(0, 0);
+ testEvenDistribution(1, 2);
+ }
+
+ private void testEvenDistribution(double width, double height)
+ {
+ Rectangle extent = new Rectangle(0, 0, 9, 4);
+ ImmutableList.Builder rectangles = ImmutableList.builder();
+ for (int x = 0; x < 10; x++) {
+ for (int y = 0; y < 5; y++) {
+ rectangles.add(new Rectangle(x, y, x + width, y + height));
+ }
+ }
+
+ KdbTree tree = buildKdbTree(10, extent, rectangles.build());
+
+ Map leafNodes = tree.getLeaves();
+ assertEquals(leafNodes.size(), 6);
+ assertEquals(leafNodes.keySet(), ImmutableSet.of(0, 1, 2, 3, 4, 5));
+ assertEquals(leafNodes.get(0), new Rectangle(0, 0, 2.5, 2.5));
+ assertEquals(leafNodes.get(1), new Rectangle(0, 2.5, 2.5, 4));
+ assertEquals(leafNodes.get(2), new Rectangle(2.5, 0, 4.5, 4));
+ assertEquals(leafNodes.get(3), new Rectangle(4.5, 0, 7.5, 2.5));
+ assertEquals(leafNodes.get(4), new Rectangle(4.5, 2.5, 7.5, 4));
+ assertEquals(leafNodes.get(5), new Rectangle(7.5, 0, 9, 4));
+ }
+
+ @Test
+ public void testSkewedDistribution()
+ {
+ testSkewedDistribution(0, 0);
+ testSkewedDistribution(1, 2);
+ }
+
+ private void testSkewedDistribution(double width, double height)
+ {
+ Rectangle extent = new Rectangle(0, 0, 9, 4);
+ ImmutableList.Builder rectangles = ImmutableList.builder();
+ for (int x = 0; x < 10; x++) {
+ for (int y = 0; y < 5; y++) {
+ rectangles.add(new Rectangle(x, y, x + width, y + height));
+ }
+ }
+
+ for (double x = 5; x < 6; x += 0.2) {
+ for (double y = 1; y < 2; y += 0.5) {
+ rectangles.add(new Rectangle(x, y, x + width, y + height));
+ }
+ }
+
+ KdbTree tree = buildKdbTree(10, extent, rectangles.build());
+
+ Map leafNodes = tree.getLeaves();
+ assertEquals(leafNodes.size(), 9);
+ assertEquals(leafNodes.keySet(), ImmutableSet.of(0, 1, 2, 3, 4, 5, 6, 7, 8));
+ assertEquals(leafNodes.get(0), new Rectangle(0, 0, 1.5, 2.5));
+ assertEquals(leafNodes.get(1), new Rectangle(1.5, 0, 3.5, 2.5));
+ assertEquals(leafNodes.get(2), new Rectangle(0, 2.5, 3.5, 4));
+ assertEquals(leafNodes.get(3), new Rectangle(3.5, 0, 5.1, 1.75));
+ assertEquals(leafNodes.get(4), new Rectangle(3.5, 1.75, 5.1, 4));
+ assertEquals(leafNodes.get(5), new Rectangle(5.1, 0, 5.9, 1.75));
+ assertEquals(leafNodes.get(6), new Rectangle(5.9, 0, 9, 1.75));
+ assertEquals(leafNodes.get(7), new Rectangle(5.1, 1.75, 7.5, 4));
+ assertEquals(leafNodes.get(8), new Rectangle(7.5, 1.75, 9, 4));
+ }
+
+ @Test
+ public void testCantSplitVertically()
+ {
+ testCantSplitVertically(0, 0);
+ testCantSplitVertically(1, 2);
+ }
+
+ private void testCantSplitVertically(double width, double height)
+ {
+ Rectangle extent = new Rectangle(0, 0, 9 + width, 4 + height);
+ ImmutableList.Builder rectangles = ImmutableList.builder();
+ for (int y = 0; y < 5; y++) {
+ for (int i = 0; i < 10; i++) {
+ rectangles.add(new Rectangle(0, y, width, y + height));
+ rectangles.add(new Rectangle(9, y, 9 + width, y + height));
+ }
+ }
+
+ KdbTree tree = buildKdbTree(10, extent, rectangles.build());
+
+ Map leafNodes = tree.getLeaves();
+ assertEquals(leafNodes.size(), 10);
+ assertEquals(leafNodes.keySet(), ImmutableSet.of(0, 1, 2, 3, 4, 5, 6, 7, 8, 9));
+ assertEquals(leafNodes.get(0), new Rectangle(0, 0, 4.5, 0.5));
+ assertEquals(leafNodes.get(1), new Rectangle(0, 0.5, 4.5, 1.5));
+ assertEquals(leafNodes.get(2), new Rectangle(0, 1.5, 4.5, 2.5));
+ assertEquals(leafNodes.get(3), new Rectangle(0, 2.5, 4.5, 3.5));
+ assertEquals(leafNodes.get(4), new Rectangle(0, 3.5, 4.5, 4 + height));
+ assertEquals(leafNodes.get(5), new Rectangle(4.5, 0, 9 + width, 0.5));
+ assertEquals(leafNodes.get(6), new Rectangle(4.5, 0.5, 9 + width, 1.5));
+ assertEquals(leafNodes.get(7), new Rectangle(4.5, 1.5, 9 + width, 2.5));
+ assertEquals(leafNodes.get(8), new Rectangle(4.5, 2.5, 9 + width, 3.5));
+ assertEquals(leafNodes.get(9), new Rectangle(4.5, 3.5, 9 + width, 4 + height));
+ }
+
+ @Test
+ public void testCantSplit()
+ {
+ testCantSplit(0, 0);
+ testCantSplit(1, 2);
+ }
+
+ private void testCantSplit(double width, double height)
+ {
+ Rectangle extent = new Rectangle(0, 0, 9 + width, 4 + height);
+ ImmutableList.Builder rectangles = ImmutableList.builder();
+ for (int i = 0; i < 10; i++) {
+ for (int j = 0; j < 5; j++) {
+ rectangles.add(new Rectangle(0, 0, width, height));
+ rectangles.add(new Rectangle(9, 4, 9 + width, 4 + height));
+ }
+ }
+
+ KdbTree tree = buildKdbTree(10, extent, rectangles.build());
+
+ Map leafNodes = tree.getLeaves();
+ assertEquals(leafNodes.size(), 2);
+ assertEquals(leafNodes.keySet(), ImmutableSet.of(0, 1));
+ assertEquals(leafNodes.get(0), new Rectangle(0, 0, 4.5, 4 + height));
+ assertEquals(leafNodes.get(1), new Rectangle(4.5, 0, 9 + width, 4 + height));
+ }
+}
diff --git a/presto-geospatial/pom.xml b/presto-geospatial/pom.xml
index a152be75eb00c..5ea84a3e02561 100644
--- a/presto-geospatial/pom.xml
+++ b/presto-geospatial/pom.xml
@@ -4,7 +4,7 @@
com.facebook.presto
presto-root
- 0.209-SNAPSHOT
+ 0.216-SNAPSHOT
presto-geospatial
@@ -31,6 +31,11 @@
jts-core
+
+ com.fasterxml.jackson.core
+ jackson-databind
+
+
com.facebook.presto
presto-geospatial-toolkit
@@ -77,21 +82,33 @@
+
+ com.facebook.presto.hadoop
+ hadoop-apache2
+ test
+
+
com.facebook.presto
- presto-parser
+ presto-hive
test
com.facebook.presto
- presto-memory
+ presto-tpch
test
- com.fasterxml.jackson.core
- jackson-databind
+ com.facebook.presto
+ presto-parser
+ test
+
+
+
+ com.facebook.presto
+ presto-memory
test
diff --git a/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/BingTile.java b/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/BingTile.java
index 986cffb6e11bd..2d098271f6337 100644
--- a/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/BingTile.java
+++ b/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/BingTile.java
@@ -51,9 +51,9 @@ public boolean equals(Object other)
}
BingTile otherTile = (BingTile) other;
- return this.x == otherTile.x
- && this.y == otherTile.y
- && this.zoomLevel == otherTile.zoomLevel;
+ return this.x == otherTile.x &&
+ this.y == otherTile.y &&
+ this.zoomLevel == otherTile.zoomLevel;
}
@Override
@@ -66,10 +66,10 @@ public int hashCode()
public String toString()
{
return toStringHelper(this)
- .add("x", x)
- .add("y", y)
- .add("zoom_level", zoomLevel)
- .toString();
+ .add("x", x)
+ .add("y", y)
+ .add("zoom_level", zoomLevel)
+ .toString();
}
@JsonCreator
diff --git a/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/BingTileFunctions.java b/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/BingTileFunctions.java
index d69d81259bd62..485a5a3ea1cd9 100644
--- a/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/BingTileFunctions.java
+++ b/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/BingTileFunctions.java
@@ -150,7 +150,7 @@ public static long toBingTile(@SqlType(StandardTypes.VARCHAR) Slice quadKey)
return BingTile.fromQuadKey(quadKey.toStringUtf8()).encode();
}
- @Description("Given a (longitude, latitude) point, returns the containing Bing tile at the specified zoom level")
+ @Description("Given a (latitude, longitude) point, returns the containing Bing tile at the specified zoom level")
@ScalarFunction("bing_tile_at")
@SqlType(BingTileType.NAME)
public static long bingTileAt(
@@ -243,7 +243,8 @@ public static Block bingTilesAround(
int totalTileCount = tileCountX * tileCountY;
checkCondition(totalTileCount <= 1_000_000,
- "The number of input tiles is too large (more than 1M) to compute a set of covering Bing tiles.");
+ "The number of tiles covering input rectangle exceeds the limit of 1M. Number of tiles: %d. Radius: %.1f km. Zoom level: %d.",
+ totalTileCount, radiusInKm, zoomLevel);
BlockBuilder blockBuilder = BIGINT.createBlockBuilder(null, totalTileCount);
@@ -371,7 +372,7 @@ public static Block geometryToBingTiles(@SqlType(GEOMETRY_TYPE_NAME) Slice input
// XY coordinates start at (0,0) in the left upper corner and increase left to right and top to bottom
long tileCount = (long) (rightLowerTile.getX() - leftUpperTile.getX() + 1) * (rightLowerTile.getY() - leftUpperTile.getY() + 1);
- checkGeometryToBingTilesLimits(ogcGeometry, pointOrRectangle, tileCount);
+ checkGeometryToBingTilesLimits(ogcGeometry, envelope, pointOrRectangle, tileCount, zoomLevel);
BlockBuilder blockBuilder = BIGINT.createBlockBuilder(null, toIntExact(tileCount));
if (pointOrRectangle || zoomLevel <= OPTIMIZED_TILING_MIN_ZOOM_LEVEL) {
@@ -427,10 +428,12 @@ private static BingTile getTileCoveringLowerRightCorner(Envelope envelope, int z
return tile;
}
- private static void checkGeometryToBingTilesLimits(OGCGeometry ogcGeometry, boolean pointOrRectangle, long tileCount)
+ private static void checkGeometryToBingTilesLimits(OGCGeometry ogcGeometry, Envelope envelope, boolean pointOrRectangle, long tileCount, int zoomLevel)
{
if (pointOrRectangle) {
- checkCondition(tileCount <= 1_000_000, "The number of input tiles is too large (more than 1M) to compute a set of covering Bing tiles.");
+ checkCondition(tileCount <= 1_000_000, "The number of tiles covering input rectangle exceeds the limit of 1M. " +
+ "Number of tiles: %d. Rectangle: xMin=%.2f, yMin=%.2f, xMax=%.2f, yMax=%.2f. Zoom level: %d.",
+ tileCount, envelope.getXMin(), envelope.getYMin(), envelope.getXMax(), envelope.getYMax(), zoomLevel);
}
else {
checkCondition((int) tileCount == tileCount, "The zoom level is too high to compute a set of covering Bing tiles.");
diff --git a/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/BingTileOperators.java b/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/BingTileOperators.java
index 66032e2e95e5d..0e78714b4e0e1 100644
--- a/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/BingTileOperators.java
+++ b/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/BingTileOperators.java
@@ -15,6 +15,7 @@
import com.facebook.presto.spi.function.IsNull;
import com.facebook.presto.spi.function.ScalarOperator;
+import com.facebook.presto.spi.function.SqlNullable;
import com.facebook.presto.spi.function.SqlType;
import com.facebook.presto.spi.type.AbstractLongType;
import com.facebook.presto.spi.type.StandardTypes;
@@ -30,14 +31,16 @@ private BingTileOperators() {}
@ScalarOperator(EQUAL)
@SqlType(StandardTypes.BOOLEAN)
- public static boolean equal(@SqlType(BingTileType.NAME) long left, @SqlType(BingTileType.NAME) long right)
+ @SqlNullable
+ public static Boolean equal(@SqlType(BingTileType.NAME) long left, @SqlType(BingTileType.NAME) long right)
{
return left == right;
}
@ScalarOperator(NOT_EQUAL)
@SqlType(StandardTypes.BOOLEAN)
- public static boolean notEqual(@SqlType(BingTileType.NAME) long left, @SqlType(BingTileType.NAME) long right)
+ @SqlNullable
+ public static Boolean notEqual(@SqlType(BingTileType.NAME) long left, @SqlType(BingTileType.NAME) long right)
{
return left != right;
}
diff --git a/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/GeoFunctions.java b/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/GeoFunctions.java
index 47c44685fea59..29afe84d70f20 100644
--- a/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/GeoFunctions.java
+++ b/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/GeoFunctions.java
@@ -15,16 +15,19 @@
import com.esri.core.geometry.Envelope;
import com.esri.core.geometry.GeometryCursor;
+import com.esri.core.geometry.ListeningGeometryCursor;
import com.esri.core.geometry.MultiPath;
import com.esri.core.geometry.MultiPoint;
import com.esri.core.geometry.MultiVertexGeometry;
import com.esri.core.geometry.NonSimpleResult;
import com.esri.core.geometry.NonSimpleResult.Reason;
import com.esri.core.geometry.OperatorSimplifyOGC;
+import com.esri.core.geometry.OperatorUnion;
import com.esri.core.geometry.Point;
import com.esri.core.geometry.Polygon;
import com.esri.core.geometry.Polyline;
import com.esri.core.geometry.SpatialReference;
+import com.esri.core.geometry.ogc.OGCConcreteGeometryCollection;
import com.esri.core.geometry.ogc.OGCGeometry;
import com.esri.core.geometry.ogc.OGCGeometryCollection;
import com.esri.core.geometry.ogc.OGCLineString;
@@ -32,6 +35,8 @@
import com.esri.core.geometry.ogc.OGCPoint;
import com.esri.core.geometry.ogc.OGCPolygon;
import com.facebook.presto.geospatial.GeometryType;
+import com.facebook.presto.geospatial.KdbTree;
+import com.facebook.presto.geospatial.Rectangle;
import com.facebook.presto.geospatial.serde.GeometrySerde;
import com.facebook.presto.geospatial.serde.GeometrySerializationType;
import com.facebook.presto.geospatial.serde.JtsGeometrySerde;
@@ -42,15 +47,23 @@
import com.facebook.presto.spi.function.ScalarFunction;
import com.facebook.presto.spi.function.SqlNullable;
import com.facebook.presto.spi.function.SqlType;
-import com.facebook.presto.spi.type.StandardTypes;
+import com.facebook.presto.spi.type.IntegerType;
import com.google.common.base.Joiner;
+import com.google.common.base.VerifyException;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import io.airlift.slice.Slice;
import org.locationtech.jts.geom.Geometry;
import org.locationtech.jts.linearref.LengthIndexedLine;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Deque;
import java.util.EnumSet;
+import java.util.Iterator;
+import java.util.List;
import java.util.Map;
+import java.util.NoSuchElementException;
import java.util.Objects;
import java.util.Set;
@@ -76,10 +89,18 @@
import static com.facebook.presto.plugin.geospatial.GeometryType.GEOMETRY;
import static com.facebook.presto.plugin.geospatial.GeometryType.GEOMETRY_TYPE_NAME;
import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT;
+import static com.facebook.presto.spi.type.StandardTypes.BIGINT;
+import static com.facebook.presto.spi.type.StandardTypes.BOOLEAN;
import static com.facebook.presto.spi.type.StandardTypes.DOUBLE;
import static com.facebook.presto.spi.type.StandardTypes.INTEGER;
+import static com.facebook.presto.spi.type.StandardTypes.TINYINT;
+import static com.facebook.presto.spi.type.StandardTypes.VARBINARY;
+import static com.facebook.presto.spi.type.StandardTypes.VARCHAR;
import static com.google.common.base.Preconditions.checkArgument;
import static io.airlift.slice.Slices.utf8Slice;
+import static io.airlift.slice.Slices.wrappedBuffer;
+import static java.lang.Double.isInfinite;
+import static java.lang.Double.isNaN;
import static java.lang.Math.atan2;
import static java.lang.Math.cos;
import static java.lang.Math.sin;
@@ -87,6 +108,8 @@
import static java.lang.Math.toIntExact;
import static java.lang.Math.toRadians;
import static java.lang.String.format;
+import static java.util.Arrays.setAll;
+import static java.util.Objects.requireNonNull;
import static org.locationtech.jts.simplify.TopologyPreservingSimplifier.simplify;
public final class GeoFunctions
@@ -104,19 +127,61 @@ public final class GeoFunctions
.put(OGCPolygonSelfTangency, "Self-tangency")
.put(OGCDisconnectedInterior, "Disconnected interior")
.build();
+ private static final int NUMBER_OF_DIMENSIONS = 3;
+ private static final Block EMPTY_ARRAY_OF_INTS = IntegerType.INTEGER.createFixedSizeBlockBuilder(0).build();
private GeoFunctions() {}
@Description("Returns a Geometry type LineString object from Well-Known Text representation (WKT)")
@ScalarFunction("ST_LineFromText")
@SqlType(GEOMETRY_TYPE_NAME)
- public static Slice parseLine(@SqlType(StandardTypes.VARCHAR) Slice input)
+ public static Slice parseLine(@SqlType(VARCHAR) Slice input)
{
OGCGeometry geometry = geometryFromText(input);
validateType("ST_LineFromText", geometry, EnumSet.of(LINE_STRING));
return serialize(geometry);
}
+ @Description("Returns a LineString from an array of points")
+ @ScalarFunction("ST_LineString")
+ @SqlType(GEOMETRY_TYPE_NAME)
+ public static Slice stLineString(@SqlType("array(" + GEOMETRY_TYPE_NAME + ")") Block input)
+ {
+ MultiPath multipath = new Polyline();
+ OGCPoint previousPoint = null;
+ for (int i = 0; i < input.getPositionCount(); i++) {
+ Slice slice = GEOMETRY.getSlice(input, i);
+
+ if (slice.getInput().available() == 0) {
+ throw new PrestoException(INVALID_FUNCTION_ARGUMENT, format("Invalid input to ST_LineString: null point at index %s", i + 1));
+ }
+
+ OGCGeometry geometry = deserialize(slice);
+ if (!(geometry instanceof OGCPoint)) {
+ throw new PrestoException(INVALID_FUNCTION_ARGUMENT, format("ST_LineString takes only an array of valid points, %s was passed", geometry.geometryType()));
+ }
+ OGCPoint point = (OGCPoint) geometry;
+
+ if (point.isEmpty()) {
+ throw new PrestoException(INVALID_FUNCTION_ARGUMENT, format("Invalid input to ST_LineString: empty point at index %s", i + 1));
+ }
+
+ if (previousPoint == null) {
+ multipath.startPath(point.X(), point.Y());
+ }
+ else {
+ if (point.Equals(previousPoint)) {
+ throw new PrestoException(INVALID_FUNCTION_ARGUMENT,
+ format("Invalid input to ST_LineString: consecutive duplicate points at index %s", i + 1));
+ }
+ multipath.lineTo(point.X(), point.Y());
+ }
+ previousPoint = point;
+ }
+ OGCLineString linestring = new OGCLineString(multipath, 0, null);
+ return serialize(linestring);
+ }
+
@Description("Returns a Geometry type Point object with the given coordinate values")
@ScalarFunction("ST_Point")
@SqlType(GEOMETRY_TYPE_NAME)
@@ -126,10 +191,40 @@ public static Slice stPoint(@SqlType(DOUBLE) double x, @SqlType(DOUBLE) double y
return serialize(geometry);
}
+ @SqlNullable
+ @Description("Returns a multi-point geometry formed from input points")
+ @ScalarFunction("ST_MultiPoint")
+ @SqlType(GEOMETRY_TYPE_NAME)
+ public static Slice stMultiPoint(@SqlType("array(" + GEOMETRY_TYPE_NAME + ")") Block input)
+ {
+ MultiPoint multipoint = new MultiPoint();
+ for (int i = 0; i < input.getPositionCount(); i++) {
+ if (input.isNull(i)) {
+ throw new PrestoException(INVALID_FUNCTION_ARGUMENT, format("Invalid input to ST_MultiPoint: null at index %s", i + 1));
+ }
+
+ Slice slice = GEOMETRY.getSlice(input, i);
+ OGCGeometry geometry = deserialize(slice);
+ if (!(geometry instanceof OGCPoint)) {
+ throw new PrestoException(INVALID_FUNCTION_ARGUMENT, format("Invalid input to ST_MultiPoint: geometry is not a point: %s at index %s", geometry.geometryType(), i + 1));
+ }
+ OGCPoint point = (OGCPoint) geometry;
+ if (point.isEmpty()) {
+ throw new PrestoException(INVALID_FUNCTION_ARGUMENT, format("Invalid input to ST_MultiPoint: empty point at index %s", i + 1));
+ }
+
+ multipoint.add(point.X(), point.Y());
+ }
+ if (multipoint.getPointCount() == 0) {
+ return null;
+ }
+ return serialize(createFromEsriGeometry(multipoint, null, true));
+ }
+
@Description("Returns a Geometry type Polygon object from Well-Known Text representation (WKT)")
@ScalarFunction("ST_Polygon")
@SqlType(GEOMETRY_TYPE_NAME)
- public static Slice stPolygon(@SqlType(StandardTypes.VARCHAR) Slice input)
+ public static Slice stPolygon(@SqlType(VARCHAR) Slice input)
{
OGCGeometry geometry = geometryFromText(input);
validateType("ST_Polygon", geometry, EnumSet.of(POLYGON));
@@ -164,26 +259,42 @@ public static double stArea(@SqlType(GEOMETRY_TYPE_NAME) Slice input)
@Description("Returns a Geometry type object from Well-Known Text representation (WKT)")
@ScalarFunction("ST_GeometryFromText")
@SqlType(GEOMETRY_TYPE_NAME)
- public static Slice stGeometryFromText(@SqlType(StandardTypes.VARCHAR) Slice input)
+ public static Slice stGeometryFromText(@SqlType(VARCHAR) Slice input)
{
return serialize(geometryFromText(input));
}
+ @Description("Returns a Geometry type object from Well-Known Binary representation (WKB)")
+ @ScalarFunction("ST_GeomFromBinary")
+ @SqlType(GEOMETRY_TYPE_NAME)
+ public static Slice stGeomFromBinary(@SqlType(VARBINARY) Slice input)
+ {
+ return serialize(geomFromBinary(input));
+ }
+
@Description("Returns the Well-Known Text (WKT) representation of the geometry")
@ScalarFunction("ST_AsText")
- @SqlType(StandardTypes.VARCHAR)
+ @SqlType(VARCHAR)
public static Slice stAsText(@SqlType(GEOMETRY_TYPE_NAME) Slice input)
{
return utf8Slice(deserialize(input).asText());
}
+ @Description("Returns the Well-Known Binary (WKB) representation of the geometry")
+ @ScalarFunction("ST_AsBinary")
+ @SqlType(VARBINARY)
+ public static Slice stAsBinary(@SqlType(GEOMETRY_TYPE_NAME) Slice input)
+ {
+ return wrappedBuffer(deserialize(input).asBinary());
+ }
+
@SqlNullable
@Description("Returns the geometry that represents all points whose distance from the specified geometry is less than or equal to the specified distance")
@ScalarFunction("ST_Buffer")
@SqlType(GEOMETRY_TYPE_NAME)
public static Slice stBuffer(@SqlType(GEOMETRY_TYPE_NAME) Slice input, @SqlType(DOUBLE) double distance)
{
- if (Double.isNaN(distance)) {
+ if (isNaN(distance)) {
throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "distance is NaN");
}
@@ -246,32 +357,18 @@ public static Slice stCentroid(@SqlType(GEOMETRY_TYPE_NAME) Slice input)
public static Slice stConvexHull(@SqlType(GEOMETRY_TYPE_NAME) Slice input)
{
OGCGeometry geometry = deserialize(input);
- validateType("ST_ConvexHull", geometry, EnumSet.of(POINT, MULTI_POINT, LINE_STRING, MULTI_LINE_STRING, POLYGON, MULTI_POLYGON));
if (geometry.isEmpty()) {
return input;
}
if (GeometryType.getForEsriGeometryType(geometry.geometryType()) == POINT) {
return input;
}
- OGCGeometry convexHull = geometry.convexHull();
- if (convexHull.isEmpty()) {
- // This happens for a single-point multi-point because of a bug in ESRI library - https://github.com/Esri/geometry-api-java/issues/172
- return serialize(createFromEsriGeometry(((MultiVertexGeometry) geometry.getEsriGeometry()).getPoint(0), null));
- }
- if (GeometryType.getForEsriGeometryType(convexHull.geometryType()) == MULTI_POLYGON) {
- MultiVertexGeometry multiVertex = (MultiVertexGeometry) convexHull.getEsriGeometry();
- if (multiVertex.getPointCount() == 2) {
- // This happens when all points of the input geometry are on the same line because of a bug in ESRI library - https://github.com/Esri/geometry-api-java/issues/172
- OGCGeometry linestring = createFromEsriGeometry(new Polyline(multiVertex.getPoint(0), multiVertex.getPoint(1)), null);
- return serialize(linestring);
- }
- }
- return serialize(convexHull);
+ return serialize(geometry.convexHull());
}
@Description("Return the coordinate dimension of the Geometry")
@ScalarFunction("ST_CoordDim")
- @SqlType(StandardTypes.TINYINT)
+ @SqlType(TINYINT)
public static long stCoordinateDimension(@SqlType(GEOMETRY_TYPE_NAME) Slice input)
{
return deserialize(input).coordinateDimension();
@@ -279,7 +376,7 @@ public static long stCoordinateDimension(@SqlType(GEOMETRY_TYPE_NAME) Slice inpu
@Description("Returns the inherent dimension of this Geometry object, which must be less than or equal to the coordinate dimension")
@ScalarFunction("ST_Dimension")
- @SqlType(StandardTypes.TINYINT)
+ @SqlType(TINYINT)
public static long stDimension(@SqlType(GEOMETRY_TYPE_NAME) Slice input)
{
return deserialize(input).dimension();
@@ -288,7 +385,7 @@ public static long stDimension(@SqlType(GEOMETRY_TYPE_NAME) Slice input)
@SqlNullable
@Description("Returns TRUE if the LineString or Multi-LineString's start and end points are coincident")
@ScalarFunction("ST_IsClosed")
- @SqlType(StandardTypes.BOOLEAN)
+ @SqlType(BOOLEAN)
public static Boolean stIsClosed(@SqlType(GEOMETRY_TYPE_NAME) Slice input)
{
OGCGeometry geometry = deserialize(input);
@@ -308,15 +405,16 @@ public static Boolean stIsClosed(@SqlType(GEOMETRY_TYPE_NAME) Slice input)
@SqlNullable
@Description("Returns TRUE if this Geometry is an empty geometrycollection, polygon, point etc")
@ScalarFunction("ST_IsEmpty")
- @SqlType(StandardTypes.BOOLEAN)
+ @SqlType(BOOLEAN)
public static Boolean stIsEmpty(@SqlType(GEOMETRY_TYPE_NAME) Slice input)
{
- return deserialize(input).isEmpty();
+ Envelope envelope = deserializeEnvelope(input);
+ return envelope == null || envelope.isEmpty();
}
@Description("Returns TRUE if this Geometry has no anomalous geometric points, such as self intersection or self tangency")
@ScalarFunction("ST_IsSimple")
- @SqlType(StandardTypes.BOOLEAN)
+ @SqlType(BOOLEAN)
public static boolean stIsSimple(@SqlType(GEOMETRY_TYPE_NAME) Slice input)
{
OGCGeometry geometry = deserialize(input);
@@ -325,7 +423,7 @@ public static boolean stIsSimple(@SqlType(GEOMETRY_TYPE_NAME) Slice input)
@Description("Returns true if the input geometry is well formed")
@ScalarFunction("ST_IsValid")
- @SqlType(StandardTypes.BOOLEAN)
+ @SqlType(BOOLEAN)
public static boolean stIsValid(@SqlType(GEOMETRY_TYPE_NAME) Slice input)
{
GeometryCursor cursor = deserialize(input).getEsriGeometryCursor();
@@ -343,7 +441,7 @@ public static boolean stIsValid(@SqlType(GEOMETRY_TYPE_NAME) Slice input)
@Description("Returns the reason for why the input geometry is not valid. Returns null if the input is valid.")
@ScalarFunction("geometry_invalid_reason")
- @SqlType(StandardTypes.VARCHAR)
+ @SqlType(VARCHAR)
@SqlNullable
public static Slice invalidReason(@SqlType(GEOMETRY_TYPE_NAME) Slice input)
{
@@ -470,7 +568,7 @@ public static Double stYMin(@SqlType(GEOMETRY_TYPE_NAME) Slice input)
@SqlNullable
@Description("Returns the cardinality of the collection of interior rings of a polygon")
@ScalarFunction("ST_NumInteriorRing")
- @SqlType(StandardTypes.BIGINT)
+ @SqlType(BIGINT)
public static Long stNumInteriorRings(@SqlType(GEOMETRY_TYPE_NAME) Slice input)
{
OGCGeometry geometry = deserialize(input);
@@ -503,7 +601,7 @@ public static Block stInteriorRings(@SqlType(GEOMETRY_TYPE_NAME) Slice input)
@Description("Returns the cardinality of the geometry collection")
@ScalarFunction("ST_NumGeometries")
- @SqlType(StandardTypes.INTEGER)
+ @SqlType(INTEGER)
public static long stNumGeometries(@SqlType(GEOMETRY_TYPE_NAME) Slice input)
{
OGCGeometry geometry = deserialize(input);
@@ -517,27 +615,66 @@ public static long stNumGeometries(@SqlType(GEOMETRY_TYPE_NAME) Slice input)
return ((OGCGeometryCollection) geometry).numGeometries();
}
- @Description("Returns a geometry that represents the point set union of the input geometries. This function doesn't support geometry collections.")
+ @Description("Returns a geometry that represents the point set union of the input geometries.")
@ScalarFunction("ST_Union")
@SqlType(GEOMETRY_TYPE_NAME)
public static Slice stUnion(@SqlType(GEOMETRY_TYPE_NAME) Slice left, @SqlType(GEOMETRY_TYPE_NAME) Slice right)
{
- // Only supports Geometry but not GeometryCollection due to ESRI library limitation
- // https://github.com/Esri/geometry-api-java/issues/176
- // https://github.com/Esri/geometry-api-java/issues/177
- OGCGeometry leftGeometry = deserialize(left);
- validateType("ST_Union", leftGeometry, EnumSet.of(POINT, MULTI_POINT, LINE_STRING, MULTI_LINE_STRING, POLYGON, MULTI_POLYGON));
- if (leftGeometry.isEmpty()) {
- return right;
+ return stUnion(ImmutableList.of(left, right));
+ }
+
+ @Description("Returns a geometry that represents the point set union of the input geometries.")
+ @ScalarFunction("geometry_union")
+ @SqlType(GEOMETRY_TYPE_NAME)
+ public static Slice geometryUnion(@SqlType("array(" + GEOMETRY_TYPE_NAME + ")") Block input)
+ {
+ return stUnion(getGeometrySlicesFromBlock(input));
+ }
+
+ private static Slice stUnion(Iterable slices)
+ {
+ // The current state of Esri/geometry-api-java does not allow support for multiple dimensions being
+ // fed to the union operator without dropping the lower dimensions:
+ // https://github.com/Esri/geometry-api-java/issues/199
+ // When operating over a collection of geometries, it is more efficient to reuse the same operator
+ // for the entire operation. Therefore, split the inputs and operators by dimension, and then union
+ // each dimension's result at the end.
+ ListeningGeometryCursor[] cursorsByDimension = new ListeningGeometryCursor[NUMBER_OF_DIMENSIONS];
+ GeometryCursor[] operatorsByDimension = new GeometryCursor[NUMBER_OF_DIMENSIONS];
+
+ setAll(cursorsByDimension, i -> new ListeningGeometryCursor());
+ setAll(operatorsByDimension, i -> OperatorUnion.local().execute(cursorsByDimension[i], null, null));
+
+ Iterator slicesIterator = slices.iterator();
+ if (!slicesIterator.hasNext()) {
+ return null;
}
+ while (slicesIterator.hasNext()) {
+ Slice slice = slicesIterator.next();
+ // Ignore null inputs
+ if (slice.getInput().available() == 0) {
+ continue;
+ }
- OGCGeometry rightGeometry = deserialize(right);
- validateType("ST_Union", rightGeometry, EnumSet.of(POINT, MULTI_POINT, LINE_STRING, MULTI_LINE_STRING, POLYGON, MULTI_POLYGON));
- if (rightGeometry.isEmpty()) {
- return left;
+ for (OGCGeometry geometry : flattenCollection(deserialize(slice))) {
+ int dimension = geometry.dimension();
+ cursorsByDimension[dimension].tick(geometry.getEsriGeometry());
+ operatorsByDimension[dimension].tock();
+ }
+ }
+
+ List outputs = new ArrayList<>();
+ for (GeometryCursor operator : operatorsByDimension) {
+ OGCGeometry unionedGeometry = createFromEsriGeometry(operator.next(), null);
+ if (unionedGeometry != null) {
+ outputs.add(unionedGeometry);
+ }
}
- return serialize(leftGeometry.union(rightGeometry));
+ if (outputs.size() == 1) {
+ return serialize(outputs.get(0));
+ }
+ return serialize(new OGCConcreteGeometryCollection(outputs, null).flattenAndRemoveOverlaps().reduceFromMulti());
}
@SqlNullable
@@ -625,7 +762,7 @@ public static Slice stInteriorRingN(@SqlType(GEOMETRY_TYPE_NAME) Slice input, @S
@Description("Returns the number of points in a Geometry")
@ScalarFunction("ST_NumPoints")
- @SqlType(StandardTypes.BIGINT)
+ @SqlType(BIGINT)
public static long stNumPoints(@SqlType(GEOMETRY_TYPE_NAME) Slice input)
{
return getPointCount(deserialize(input));
@@ -634,7 +771,7 @@ public static long stNumPoints(@SqlType(GEOMETRY_TYPE_NAME) Slice input)
@SqlNullable
@Description("Returns TRUE if and only if the line is closed and simple")
@ScalarFunction("ST_IsRing")
- @SqlType(StandardTypes.BOOLEAN)
+ @SqlType(BOOLEAN)
public static Boolean stIsRing(@SqlType(GEOMETRY_TYPE_NAME) Slice input)
{
OGCGeometry geometry = deserialize(input);
@@ -662,10 +799,9 @@ public static Slice stStartPoint(@SqlType(GEOMETRY_TYPE_NAME) Slice input)
@Description("Returns a \"simplified\" version of the given geometry")
@ScalarFunction("simplify_geometry")
@SqlType(GEOMETRY_TYPE_NAME)
- public static Slice simplifyGeometry(@SqlType(GEOMETRY_TYPE_NAME) Slice input,
- @SqlType(DOUBLE) double distanceTolerance)
+ public static Slice simplifyGeometry(@SqlType(GEOMETRY_TYPE_NAME) Slice input, @SqlType(DOUBLE) double distanceTolerance)
{
- if (Double.isNaN(distanceTolerance)) {
+ if (isNaN(distanceTolerance)) {
throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "distanceTolerance is NaN");
}
@@ -778,15 +914,16 @@ public static Slice stDifference(@SqlType(GEOMETRY_TYPE_NAME) Slice left, @SqlTy
return serialize(leftGeometry.difference(rightGeometry));
}
+ @SqlNullable
@Description("Returns the 2-dimensional cartesian minimum distance (based on spatial ref) between two geometries in projected units")
@ScalarFunction("ST_Distance")
@SqlType(DOUBLE)
- public static double stDistance(@SqlType(GEOMETRY_TYPE_NAME) Slice left, @SqlType(GEOMETRY_TYPE_NAME) Slice right)
+ public static Double stDistance(@SqlType(GEOMETRY_TYPE_NAME) Slice left, @SqlType(GEOMETRY_TYPE_NAME) Slice right)
{
OGCGeometry leftGeometry = deserialize(left);
OGCGeometry rightGeometry = deserialize(right);
verifySameSpatialReference(leftGeometry, rightGeometry);
- return leftGeometry.distance(rightGeometry);
+ return leftGeometry.isEmpty() || rightGeometry.isEmpty() ? null : leftGeometry.distance(rightGeometry);
}
@SqlNullable
@@ -852,7 +989,7 @@ public static Slice stSymmetricDifference(@SqlType(GEOMETRY_TYPE_NAME) Slice lef
@SqlNullable
@Description("Returns TRUE if and only if no points of right lie in the exterior of left, and at least one point of the interior of left lies in the interior of right")
@ScalarFunction("ST_Contains")
- @SqlType(StandardTypes.BOOLEAN)
+ @SqlType(BOOLEAN)
public static Boolean stContains(@SqlType(GEOMETRY_TYPE_NAME) Slice left, @SqlType(GEOMETRY_TYPE_NAME) Slice right)
{
if (!envelopes(left, right, Envelope::contains)) {
@@ -867,7 +1004,7 @@ public static Boolean stContains(@SqlType(GEOMETRY_TYPE_NAME) Slice left, @SqlTy
@SqlNullable
@Description("Returns TRUE if the supplied geometries have some, but not all, interior points in common")
@ScalarFunction("ST_Crosses")
- @SqlType(StandardTypes.BOOLEAN)
+ @SqlType(BOOLEAN)
public static Boolean stCrosses(@SqlType(GEOMETRY_TYPE_NAME) Slice left, @SqlType(GEOMETRY_TYPE_NAME) Slice right)
{
if (!envelopes(left, right, Envelope::intersect)) {
@@ -882,7 +1019,7 @@ public static Boolean stCrosses(@SqlType(GEOMETRY_TYPE_NAME) Slice left, @SqlTyp
@SqlNullable
@Description("Returns TRUE if the Geometries do not spatially intersect - if they do not share any space together")
@ScalarFunction("ST_Disjoint")
- @SqlType(StandardTypes.BOOLEAN)
+ @SqlType(BOOLEAN)
public static Boolean stDisjoint(@SqlType(GEOMETRY_TYPE_NAME) Slice left, @SqlType(GEOMETRY_TYPE_NAME) Slice right)
{
if (!envelopes(left, right, Envelope::intersect)) {
@@ -897,7 +1034,7 @@ public static Boolean stDisjoint(@SqlType(GEOMETRY_TYPE_NAME) Slice left, @SqlTy
@SqlNullable
@Description("Returns TRUE if the given geometries represent the same geometry")
@ScalarFunction("ST_Equals")
- @SqlType(StandardTypes.BOOLEAN)
+ @SqlType(BOOLEAN)
public static Boolean stEquals(@SqlType(GEOMETRY_TYPE_NAME) Slice left, @SqlType(GEOMETRY_TYPE_NAME) Slice right)
{
OGCGeometry leftGeometry = deserialize(left);
@@ -909,7 +1046,7 @@ public static Boolean stEquals(@SqlType(GEOMETRY_TYPE_NAME) Slice left, @SqlType
@SqlNullable
@Description("Returns TRUE if the Geometries spatially intersect in 2D - (share any portion of space) and FALSE if they don't (they are Disjoint)")
@ScalarFunction("ST_Intersects")
- @SqlType(StandardTypes.BOOLEAN)
+ @SqlType(BOOLEAN)
public static Boolean stIntersects(@SqlType(GEOMETRY_TYPE_NAME) Slice left, @SqlType(GEOMETRY_TYPE_NAME) Slice right)
{
if (!envelopes(left, right, Envelope::intersect)) {
@@ -924,7 +1061,7 @@ public static Boolean stIntersects(@SqlType(GEOMETRY_TYPE_NAME) Slice left, @Sql
@SqlNullable
@Description("Returns TRUE if the Geometries share space, are of the same dimension, but are not completely contained by each other")
@ScalarFunction("ST_Overlaps")
- @SqlType(StandardTypes.BOOLEAN)
+ @SqlType(BOOLEAN)
public static Boolean stOverlaps(@SqlType(GEOMETRY_TYPE_NAME) Slice left, @SqlType(GEOMETRY_TYPE_NAME) Slice right)
{
if (!envelopes(left, right, Envelope::intersect)) {
@@ -939,8 +1076,8 @@ public static Boolean stOverlaps(@SqlType(GEOMETRY_TYPE_NAME) Slice left, @SqlTy
@SqlNullable
@Description("Returns TRUE if this Geometry is spatially related to another Geometry")
@ScalarFunction("ST_Relate")
- @SqlType(StandardTypes.BOOLEAN)
- public static Boolean stRelate(@SqlType(GEOMETRY_TYPE_NAME) Slice left, @SqlType(GEOMETRY_TYPE_NAME) Slice right, @SqlType(StandardTypes.VARCHAR) Slice relation)
+ @SqlType(BOOLEAN)
+ public static Boolean stRelate(@SqlType(GEOMETRY_TYPE_NAME) Slice left, @SqlType(GEOMETRY_TYPE_NAME) Slice right, @SqlType(VARCHAR) Slice relation)
{
OGCGeometry leftGeometry = deserialize(left);
OGCGeometry rightGeometry = deserialize(right);
@@ -951,7 +1088,7 @@ public static Boolean stRelate(@SqlType(GEOMETRY_TYPE_NAME) Slice left, @SqlType
@SqlNullable
@Description("Returns TRUE if the geometries have at least one point in common, but their interiors do not intersect")
@ScalarFunction("ST_Touches")
- @SqlType(StandardTypes.BOOLEAN)
+ @SqlType(BOOLEAN)
public static Boolean stTouches(@SqlType(GEOMETRY_TYPE_NAME) Slice left, @SqlType(GEOMETRY_TYPE_NAME) Slice right)
{
if (!envelopes(left, right, Envelope::intersect)) {
@@ -966,7 +1103,7 @@ public static Boolean stTouches(@SqlType(GEOMETRY_TYPE_NAME) Slice left, @SqlTyp
@SqlNullable
@Description("Returns TRUE if the geometry A is completely inside geometry B")
@ScalarFunction("ST_Within")
- @SqlType(StandardTypes.BOOLEAN)
+ @SqlType(BOOLEAN)
public static Boolean stWithin(@SqlType(GEOMETRY_TYPE_NAME) Slice left, @SqlType(GEOMETRY_TYPE_NAME) Slice right)
{
if (!envelopes(right, left, Envelope::contains)) {
@@ -980,20 +1117,93 @@ public static Boolean stWithin(@SqlType(GEOMETRY_TYPE_NAME) Slice left, @SqlType
@Description("Returns the type of the geometry")
@ScalarFunction("ST_GeometryType")
- @SqlType(StandardTypes.VARCHAR)
+ @SqlType(VARCHAR)
public static Slice stGeometryType(@SqlType(GEOMETRY_TYPE_NAME) Slice input)
{
return GeometrySerde.getGeometryType(input).standardName();
}
+ @ScalarFunction
+ @SqlNullable
+ @Description("Returns an array of spatial partition IDs for a given geometry")
+ @SqlType("array(int)")
+ public static Block spatialPartitions(@SqlType(KdbTreeType.NAME) Object kdbTree, @SqlType(GEOMETRY_TYPE_NAME) Slice geometry)
+ {
+ Envelope envelope = deserializeEnvelope(geometry);
+ if (envelope == null) {
+ // Empty geometry
+ return null;
+ }
+
+ return spatialPartitions((KdbTree) kdbTree, new Rectangle(envelope.getXMin(), envelope.getYMin(), envelope.getXMax(), envelope.getYMax()));
+ }
+
+ @ScalarFunction
+ @SqlNullable
+ @Description("Returns an array of spatial partition IDs for a geometry representing a set of points within specified distance from the input geometry")
+ @SqlType("array(int)")
+ public static Block spatialPartitions(@SqlType(KdbTreeType.NAME) Object kdbTree, @SqlType(GEOMETRY_TYPE_NAME) Slice geometry, @SqlType(DOUBLE) double distance)
+ {
+ if (isNaN(distance)) {
+ throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "distance is NaN");
+ }
+
+ if (isInfinite(distance)) {
+ throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "distance is infinite");
+ }
+
+ if (distance < 0) {
+ throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "distance is negative");
+ }
+
+ Envelope envelope = deserializeEnvelope(geometry);
+ if (envelope == null) {
+ return null;
+ }
+
+ Rectangle expandedEnvelope2D = new Rectangle(envelope.getXMin() - distance, envelope.getYMin() - distance, envelope.getXMax() + distance, envelope.getYMax() + distance);
+ return spatialPartitions((KdbTree) kdbTree, expandedEnvelope2D);
+ }
+
+ private static Block spatialPartitions(KdbTree kdbTree, Rectangle envelope)
+ {
+ Map partitions = kdbTree.findIntersectingLeaves(envelope);
+ if (partitions.isEmpty()) {
+ return EMPTY_ARRAY_OF_INTS;
+ }
+
+ // For input rectangles that represent a single point, return at most one partition
+ // by excluding right and upper sides of partition rectangles. The logic that builds
+ // KDB tree needs to make sure to add some padding to the right and upper sides of the
+ // overall extent of the tree to avoid missing right-most and top-most points.
+ boolean point = (envelope.getWidth() == 0 && envelope.getHeight() == 0);
+ if (point) {
+ for (Map.Entry partition : partitions.entrySet()) {
+ if (envelope.getXMin() < partition.getValue().getXMax() && envelope.getYMin() < partition.getValue().getYMax()) {
+ BlockBuilder blockBuilder = IntegerType.INTEGER.createFixedSizeBlockBuilder(1);
+ blockBuilder.writeInt(partition.getKey());
+ return blockBuilder.build();
+ }
+ }
+ throw new VerifyException(format("Cannot find half-open partition extent for a point: (%s, %s)", envelope.getXMin(), envelope.getYMin()));
+ }
+
+ BlockBuilder blockBuilder = IntegerType.INTEGER.createFixedSizeBlockBuilder(partitions.size());
+ for (int id : partitions.keySet()) {
+ blockBuilder.writeInt(id);
+ }
+
+ return blockBuilder.build();
+ }
+
@ScalarFunction
@Description("Calculates the great-circle distance between two points on the Earth's surface in kilometers")
- @SqlType(StandardTypes.DOUBLE)
+ @SqlType(DOUBLE)
public static double greatCircleDistance(
- @SqlType(StandardTypes.DOUBLE) double latitude1,
- @SqlType(StandardTypes.DOUBLE) double longitude1,
- @SqlType(StandardTypes.DOUBLE) double latitude2,
- @SqlType(StandardTypes.DOUBLE) double longitude2)
+ @SqlType(DOUBLE) double latitude1,
+ @SqlType(DOUBLE) double longitude1,
+ @SqlType(DOUBLE) double latitude2,
+ @SqlType(DOUBLE) double longitude2)
{
checkLatitude(latitude1);
checkLongitude(longitude1);
@@ -1019,14 +1229,14 @@ public static double greatCircleDistance(
private static void checkLatitude(double latitude)
{
- if (Double.isNaN(latitude) || Double.isInfinite(latitude) || latitude < -90 || latitude > 90) {
+ if (isNaN(latitude) || isInfinite(latitude) || latitude < -90 || latitude > 90) {
throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "Latitude must be between -90 and 90");
}
}
private static void checkLongitude(double longitude)
{
- if (Double.isNaN(longitude) || Double.isInfinite(longitude) || longitude < -180 || longitude > 180) {
+ if (isNaN(longitude) || isInfinite(longitude) || longitude < -180 || longitude > 180) {
throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "Longitude must be between -180 and 180");
}
}
@@ -1044,6 +1254,20 @@ private static OGCGeometry geometryFromText(Slice input)
return geometry;
}
+ private static OGCGeometry geomFromBinary(Slice input)
+ {
+ requireNonNull(input, "input is null");
+ OGCGeometry geometry;
+ try {
+ geometry = OGCGeometry.fromBinary(input.toByteBuffer().slice());
+ }
+ catch (IllegalArgumentException | IndexOutOfBoundsException e) {
+ throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "Invalid WKB", e);
+ }
+ geometry.setSpatialReference(null);
+ return geometry;
+ }
+
private static void validateType(String function, OGCGeometry geometry, Set validTypes)
{
GeometryType type = GeometryType.getForEsriGeometryType(geometry.geometryType());
@@ -1184,4 +1408,77 @@ private interface EnvelopesPredicate
{
boolean apply(Envelope left, Envelope right);
}
+
+ private static Iterable getGeometrySlicesFromBlock(Block block)
+ {
+ requireNonNull(block, "block is null");
+ return () -> new Iterator()
+ {
+ private int iteratorPosition;
+
+ @Override
+ public boolean hasNext()
+ {
+ return iteratorPosition != block.getPositionCount();
+ }
+
+ @Override
+ public Slice next()
+ {
+ if (!hasNext()) {
+ throw new NoSuchElementException("Slices have been consumed");
+ }
+ return GEOMETRY.getSlice(block, iteratorPosition++);
+ }
+ };
+ }
+
+ private static Iterable flattenCollection(OGCGeometry geometry)
+ {
+ if (geometry == null) {
+ return ImmutableList.of();
+ }
+ if (!(geometry instanceof OGCConcreteGeometryCollection)) {
+ return ImmutableList.of(geometry);
+ }
+ if (((OGCConcreteGeometryCollection) geometry).numGeometries() == 0) {
+ return ImmutableList.of();
+ }
+ return () -> new GeometryCollectionIterator(geometry);
+ }
+
+ private static class GeometryCollectionIterator
+ implements Iterator
+ {
+ private final Deque geometriesDeque = new ArrayDeque<>();
+
+ GeometryCollectionIterator(OGCGeometry geometries)
+ {
+ geometriesDeque.push(requireNonNull(geometries, "geometries is null"));
+ }
+
+ @Override
+ public boolean hasNext()
+ {
+ if (geometriesDeque.isEmpty()) {
+ return false;
+ }
+ while (geometriesDeque.peek() instanceof OGCConcreteGeometryCollection) {
+ OGCGeometryCollection collection = (OGCGeometryCollection) geometriesDeque.pop();
+ for (int i = 0; i < collection.numGeometries(); i++) {
+ geometriesDeque.push(collection.geometryN(i));
+ }
+ }
+ return !geometriesDeque.isEmpty();
+ }
+
+ @Override
+ public OGCGeometry next()
+ {
+ if (!hasNext()) {
+ throw new NoSuchElementException("Geometries have been consumed");
+ }
+ return geometriesDeque.pop();
+ }
+ }
}
diff --git a/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/GeoPlugin.java b/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/GeoPlugin.java
index 6f7d8d4a32163..396a1ae5cec06 100644
--- a/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/GeoPlugin.java
+++ b/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/GeoPlugin.java
@@ -25,6 +25,7 @@
import static com.facebook.presto.plugin.geospatial.BingTileType.BING_TILE;
import static com.facebook.presto.plugin.geospatial.GeometryType.GEOMETRY;
+import static com.facebook.presto.plugin.geospatial.KdbTreeType.KDB_TREE;
public class GeoPlugin
implements Plugin
@@ -32,7 +33,7 @@ public class GeoPlugin
@Override
public Iterable getTypes()
{
- return ImmutableList.of(GEOMETRY, BING_TILE);
+ return ImmutableList.of(GEOMETRY, BING_TILE, KDB_TREE);
}
@Override
@@ -45,6 +46,9 @@ public Set> getFunctions()
.add(BingTileCoordinatesFunction.class)
.add(ConvexHullAggregation.class)
.add(GeometryUnionAgg.class)
+ .add(KdbTreeCasts.class)
+ .add(SpatialPartitioningAggregateFunction.class)
+ .add(SpatialPartitioningInternalAggregateFunction.class)
.build();
}
}
diff --git a/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/GeometryType.java b/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/GeometryType.java
index d92d3b462a558..2b007eb380739 100644
--- a/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/GeometryType.java
+++ b/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/GeometryType.java
@@ -59,12 +59,20 @@ public Slice getSlice(Block block, int position)
@Override
public void writeSlice(BlockBuilder blockBuilder, Slice value)
{
+ if (value == null) {
+ blockBuilder.appendNull();
+ return;
+ }
writeSlice(blockBuilder, value, 0, value.length());
}
@Override
public void writeSlice(BlockBuilder blockBuilder, Slice value, int offset, int length)
{
+ if (value == null) {
+ blockBuilder.appendNull();
+ return;
+ }
blockBuilder.writeBytes(value, offset, length).closeEntry();
}
diff --git a/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/KdbTreeCasts.java b/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/KdbTreeCasts.java
new file mode 100644
index 0000000000000..b92e85d367249
--- /dev/null
+++ b/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/KdbTreeCasts.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.facebook.presto.plugin.geospatial;
+
+import com.facebook.presto.geospatial.KdbTreeUtils;
+import com.facebook.presto.spi.PrestoException;
+import com.facebook.presto.spi.function.LiteralParameters;
+import com.facebook.presto.spi.function.ScalarOperator;
+import com.facebook.presto.spi.function.SqlType;
+import io.airlift.slice.Slice;
+
+import static com.facebook.presto.spi.StandardErrorCode.INVALID_CAST_ARGUMENT;
+import static com.facebook.presto.spi.function.OperatorType.CAST;
+
+public final class KdbTreeCasts
+{
+ private KdbTreeCasts() {}
+
+ @LiteralParameters("x")
+ @ScalarOperator(CAST)
+ @SqlType(KdbTreeType.NAME)
+ public static Object castVarcharToKdbTree(@SqlType("varchar(x)") Slice json)
+ {
+ try {
+ return KdbTreeUtils.fromJson(json.toStringUtf8());
+ }
+ catch (IllegalArgumentException e) {
+ throw new PrestoException(INVALID_CAST_ARGUMENT, "Invalid JSON string for KDB tree", e);
+ }
+ }
+}
diff --git a/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/KdbTreeType.java b/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/KdbTreeType.java
new file mode 100644
index 0000000000000..a44bd85acf427
--- /dev/null
+++ b/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/KdbTreeType.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.facebook.presto.plugin.geospatial;
+
+import com.facebook.presto.spi.ConnectorSession;
+import com.facebook.presto.spi.PrestoException;
+import com.facebook.presto.spi.block.Block;
+import com.facebook.presto.spi.block.BlockBuilder;
+import com.facebook.presto.spi.block.BlockBuilderStatus;
+import com.facebook.presto.spi.type.AbstractType;
+import com.facebook.presto.spi.type.TypeSignature;
+
+import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR;
+
+public final class KdbTreeType
+ extends AbstractType
+{
+ public static final KdbTreeType KDB_TREE = new KdbTreeType();
+ public static final String NAME = "KdbTree";
+
+ private KdbTreeType()
+ {
+ super(new TypeSignature(NAME), Object.class);
+ }
+
+ @Override
+ public Object getObjectValue(ConnectorSession session, Block block, int position)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void appendTo(Block block, int position, BlockBuilder blockBuilder)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public BlockBuilder createBlockBuilder(BlockBuilderStatus blockBuilderStatus, int expectedEntries, int expectedBytesPerEntry)
+ {
+ throw new PrestoException(GENERIC_INTERNAL_ERROR, "KdbTree type cannot be serialized");
+ }
+
+ @Override
+ public BlockBuilder createBlockBuilder(BlockBuilderStatus blockBuilderStatus, int expectedEntries)
+ {
+ throw new PrestoException(GENERIC_INTERNAL_ERROR, "KdbTree type cannot be serialized");
+ }
+}
diff --git a/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/SpatialPartitioningAggregateFunction.java b/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/SpatialPartitioningAggregateFunction.java
new file mode 100644
index 0000000000000..3b79e882ea2b5
--- /dev/null
+++ b/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/SpatialPartitioningAggregateFunction.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.facebook.presto.plugin.geospatial;
+
+import com.facebook.presto.spi.block.BlockBuilder;
+import com.facebook.presto.spi.function.AggregationFunction;
+import com.facebook.presto.spi.function.CombineFunction;
+import com.facebook.presto.spi.function.InputFunction;
+import com.facebook.presto.spi.function.OutputFunction;
+import com.facebook.presto.spi.function.SqlType;
+import com.facebook.presto.spi.type.StandardTypes;
+import io.airlift.slice.Slice;
+
+import static com.facebook.presto.plugin.geospatial.GeometryType.GEOMETRY_TYPE_NAME;
+import static com.facebook.presto.plugin.geospatial.SpatialPartitioningAggregateFunction.NAME;
+
+@AggregationFunction(value = NAME, decomposable = false)
+public class SpatialPartitioningAggregateFunction
+{
+ public static final String NAME = "spatial_partitioning";
+
+ private SpatialPartitioningAggregateFunction() {}
+
+ @InputFunction
+ public static void input(SpatialPartitioningState state, @SqlType(GEOMETRY_TYPE_NAME) Slice slice)
+ {
+ throw new UnsupportedOperationException("spatial_partitioning(geometry, samplingPercentage) aggregate function should be re-written into spatial_partitioning(geometry, samplingPercentage, partitionCount)");
+ }
+
+ @CombineFunction
+ public static void combine(SpatialPartitioningState state, SpatialPartitioningState otherState)
+ {
+ throw new UnsupportedOperationException("spatial_partitioning(geometry, samplingPercentage) aggregate function should be re-written into spatial_partitioning(geometry, samplingPercentage, partitionCount)");
+ }
+
+ @OutputFunction(StandardTypes.VARCHAR)
+ public static void output(SpatialPartitioningState state, BlockBuilder out)
+ {
+ throw new UnsupportedOperationException("spatial_partitioning(geometry, samplingPercentage) aggregate function should be re-written into spatial_partitioning(geometry, samplingPercentage, partitionCount)");
+ }
+}
diff --git a/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/SpatialPartitioningInternalAggregateFunction.java b/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/SpatialPartitioningInternalAggregateFunction.java
new file mode 100644
index 0000000000000..9d4ad04b87f5c
--- /dev/null
+++ b/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/SpatialPartitioningInternalAggregateFunction.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.facebook.presto.plugin.geospatial;
+
+import com.esri.core.geometry.Envelope;
+import com.facebook.presto.geospatial.KdbTreeUtils;
+import com.facebook.presto.geospatial.Rectangle;
+import com.facebook.presto.spi.block.BlockBuilder;
+import com.facebook.presto.spi.function.AggregationFunction;
+import com.facebook.presto.spi.function.CombineFunction;
+import com.facebook.presto.spi.function.InputFunction;
+import com.facebook.presto.spi.function.OutputFunction;
+import com.facebook.presto.spi.function.SqlType;
+import com.facebook.presto.spi.type.StandardTypes;
+import io.airlift.slice.Slice;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ThreadLocalRandom;
+
+import static com.facebook.presto.geospatial.KdbTree.buildKdbTree;
+import static com.facebook.presto.geospatial.serde.GeometrySerde.deserializeEnvelope;
+import static com.facebook.presto.plugin.geospatial.GeometryType.GEOMETRY_TYPE_NAME;
+import static com.facebook.presto.plugin.geospatial.SpatialPartitioningAggregateFunction.NAME;
+import static com.facebook.presto.spi.type.StandardTypes.INTEGER;
+import static com.facebook.presto.spi.type.VarcharType.VARCHAR;
+import static java.lang.Math.toIntExact;
+
+@AggregationFunction(value = NAME, decomposable = false, hidden = true)
+public class SpatialPartitioningInternalAggregateFunction
+{
+ private static final int MAX_SAMPLE_COUNT = 1_000_000;
+
+ private SpatialPartitioningInternalAggregateFunction() {}
+
+ @InputFunction
+ public static void input(SpatialPartitioningState state, @SqlType(GEOMETRY_TYPE_NAME) Slice slice, @SqlType(INTEGER) long partitionCount)
+ {
+ Envelope envelope = deserializeEnvelope(slice);
+ if (envelope == null) {
+ return;
+ }
+
+ Rectangle extent = new Rectangle(envelope.getXMin(), envelope.getYMin(), envelope.getXMax(), envelope.getYMax());
+
+ if (state.getCount() == 0) {
+ state.setPartitionCount(toIntExact(partitionCount));
+ state.setExtent(extent);
+ state.setSamples(new ArrayList<>());
+ }
+ else {
+ state.setExtent(state.getExtent().merge(extent));
+ }
+
+ // use reservoir sampling
+ List samples = state.getSamples();
+ if (samples.size() <= MAX_SAMPLE_COUNT) {
+ samples.add(extent);
+ }
+ else {
+ long sampleIndex = ThreadLocalRandom.current().nextLong(state.getCount());
+ if (sampleIndex < MAX_SAMPLE_COUNT) {
+ samples.set(toIntExact(sampleIndex), extent);
+ }
+ }
+
+ state.setCount(state.getCount() + 1);
+ }
+
+ @CombineFunction
+ public static void combine(SpatialPartitioningState state, SpatialPartitioningState otherState)
+ {
+ throw new UnsupportedOperationException("spatial_partitioning must run on a single node");
+ }
+
+ @OutputFunction(StandardTypes.VARCHAR)
+ public static void output(SpatialPartitioningState state, BlockBuilder out)
+ {
+ if (state.getCount() == 0) {
+ out.appendNull();
+ return;
+ }
+
+ List samples = state.getSamples();
+
+ int partitionCount = state.getPartitionCount();
+ int maxItemsPerNode = (samples.size() + partitionCount - 1) / partitionCount;
+ Rectangle envelope = state.getExtent();
+
+ // Add a small buffer on the right and upper sides
+ Rectangle paddedExtent = new Rectangle(envelope.getXMin(), envelope.getYMin(), Math.nextUp(envelope.getXMax()), Math.nextUp(envelope.getYMax()));
+
+ VARCHAR.writeString(out, KdbTreeUtils.toJson(buildKdbTree(maxItemsPerNode, paddedExtent, samples)));
+ }
+}
diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/MultiKeyValuePairsState.java b/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/SpatialPartitioningState.java
similarity index 55%
rename from presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/MultiKeyValuePairsState.java
rename to presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/SpatialPartitioningState.java
index 2e50d57c24c76..fa54c2878f11b 100644
--- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/MultiKeyValuePairsState.java
+++ b/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/SpatialPartitioningState.java
@@ -11,24 +11,31 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.facebook.presto.operator.aggregation.state;
+package com.facebook.presto.plugin.geospatial;
-import com.facebook.presto.operator.aggregation.MultiKeyValuePairs;
+import com.facebook.presto.geospatial.Rectangle;
import com.facebook.presto.spi.function.AccumulatorState;
import com.facebook.presto.spi.function.AccumulatorStateMetadata;
-import com.facebook.presto.spi.type.Type;
-@AccumulatorStateMetadata(stateFactoryClass = MultiKeyValuePairsStateFactory.class, stateSerializerClass = MultiKeyValuePairStateSerializer.class)
-public interface MultiKeyValuePairsState
+import java.util.List;
+
+@AccumulatorStateMetadata(stateSerializerClass = SpatialPartitioningStateSerializer.class, stateFactoryClass = SpatialPartitioningStateFactory.class)
+public interface SpatialPartitioningState
extends AccumulatorState
{
- MultiKeyValuePairs get();
+ int getPartitionCount();
+
+ void setPartitionCount(int partitionCount);
+
+ long getCount();
+
+ void setCount(long count);
- void set(MultiKeyValuePairs value);
+ Rectangle getExtent();
- void addMemoryUsage(long memory);
+ void setExtent(Rectangle envelope);
- Type getKeyType();
+ List getSamples();
- Type getValueType();
+ void setSamples(List samples);
}
diff --git a/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/SpatialPartitioningStateFactory.java b/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/SpatialPartitioningStateFactory.java
new file mode 100644
index 0000000000000..bc37ab6c3c747
--- /dev/null
+++ b/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/SpatialPartitioningStateFactory.java
@@ -0,0 +1,213 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.facebook.presto.plugin.geospatial;
+
+import com.esri.core.geometry.Envelope;
+import com.facebook.presto.array.IntBigArray;
+import com.facebook.presto.array.LongBigArray;
+import com.facebook.presto.array.ObjectBigArray;
+import com.facebook.presto.geospatial.Rectangle;
+import com.facebook.presto.spi.function.AccumulatorStateFactory;
+import com.facebook.presto.spi.function.GroupedAccumulatorState;
+import org.openjdk.jol.info.ClassLayout;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static java.lang.Math.toIntExact;
+
+public class SpatialPartitioningStateFactory
+ implements AccumulatorStateFactory
+{
+ @Override
+ public SpatialPartitioningState createSingleState()
+ {
+ return new SingleSpatialPartitioningState();
+ }
+
+ @Override
+ public Class getSingleStateClass()
+ {
+ return SpatialPartitioningState.class;
+ }
+
+ @Override
+ public SpatialPartitioningState createGroupedState()
+ {
+ return new GroupedSpatialPartitioningState();
+ }
+
+ @Override
+ public Class getGroupedStateClass()
+ {
+ return GroupedSpatialPartitioningState.class;
+ }
+
+ public static final class GroupedSpatialPartitioningState
+ implements GroupedAccumulatorState, SpatialPartitioningState
+ {
+ private static final int INSTANCE_SIZE = ClassLayout.parseClass(GroupedSpatialPartitioningState.class).instanceSize();
+ private static final int ENVELOPE_SIZE = toIntExact(new Envelope(1, 2, 3, 4).estimateMemorySize());
+
+ private long groupId;
+ private final IntBigArray partitionCounts = new IntBigArray();
+ private final LongBigArray counts = new LongBigArray();
+ private final ObjectBigArray envelopes = new ObjectBigArray<>();
+ private final ObjectBigArray> samples = new ObjectBigArray<>();
+ private int envelopeCount;
+ private int samplesCount;
+
+ @Override
+ public int getPartitionCount()
+ {
+ return partitionCounts.get(groupId);
+ }
+
+ @Override
+ public void setPartitionCount(int partitionCount)
+ {
+ this.partitionCounts.set(groupId, partitionCount);
+ }
+
+ @Override
+ public long getCount()
+ {
+ return counts.get(groupId);
+ }
+
+ @Override
+ public void setCount(long count)
+ {
+ counts.set(groupId, count);
+ }
+
+ @Override
+ public Rectangle getExtent()
+ {
+ return envelopes.get(groupId);
+ }
+
+ @Override
+ public void setExtent(Rectangle envelope)
+ {
+ if (envelopes.get(groupId) == null) {
+ envelopeCount++;
+ }
+ envelopes.set(groupId, envelope);
+ }
+
+ @Override
+ public List getSamples()
+ {
+ return samples.get(groupId);
+ }
+
+ @Override
+ public void setSamples(List samples)
+ {
+ List currentSamples = this.samples.get(groupId);
+ if (currentSamples != null) {
+ samplesCount -= currentSamples.size();
+ }
+ samplesCount += samples.size();
+ this.samples.set(groupId, samples);
+ }
+
+ @Override
+ public long getEstimatedSize()
+ {
+ return INSTANCE_SIZE + partitionCounts.sizeOf() + counts.sizeOf() + envelopes.sizeOf() + samples.sizeOf() + ENVELOPE_SIZE * (envelopeCount + samplesCount);
+ }
+
+ @Override
+ public void setGroupId(long groupId)
+ {
+ this.groupId = groupId;
+ }
+
+ @Override
+ public void ensureCapacity(long size)
+ {
+ partitionCounts.ensureCapacity(size);
+ counts.ensureCapacity(size);
+ envelopes.ensureCapacity(size);
+ samples.ensureCapacity(size);
+ }
+ }
+
+ public static final class SingleSpatialPartitioningState
+ implements SpatialPartitioningState
+ {
+ private static final int INSTANCE_SIZE = ClassLayout.parseClass(SingleSpatialPartitioningState.class).instanceSize();
+
+ private int partitionCount;
+ private long count;
+ private Rectangle envelope;
+ private List samples = new ArrayList<>();
+
+ @Override
+ public int getPartitionCount()
+ {
+ return partitionCount;
+ }
+
+ @Override
+ public void setPartitionCount(int partitionCount)
+ {
+ this.partitionCount = partitionCount;
+ }
+
+ @Override
+ public long getCount()
+ {
+ return count;
+ }
+
+ @Override
+ public void setCount(long count)
+ {
+ this.count = count;
+ }
+
+ @Override
+ public Rectangle getExtent()
+ {
+ return envelope;
+ }
+
+ @Override
+ public void setExtent(Rectangle envelope)
+ {
+ this.envelope = envelope;
+ }
+
+ @Override
+ public List getSamples()
+ {
+ return samples;
+ }
+
+ @Override
+ public void setSamples(List samples)
+ {
+ this.samples = samples;
+ }
+
+ @Override
+ public long getEstimatedSize()
+ {
+ return INSTANCE_SIZE + (envelope != null ? envelope.estimateMemorySize() * (1 + samples.size()) : 0);
+ }
+ }
+}
diff --git a/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/SpatialPartitioningStateSerializer.java b/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/SpatialPartitioningStateSerializer.java
new file mode 100644
index 0000000000000..39f62e0c779a8
--- /dev/null
+++ b/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/SpatialPartitioningStateSerializer.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.facebook.presto.plugin.geospatial;
+
+import com.facebook.presto.spi.block.Block;
+import com.facebook.presto.spi.block.BlockBuilder;
+import com.facebook.presto.spi.function.AccumulatorStateSerializer;
+import com.facebook.presto.spi.type.Type;
+
+import static com.facebook.presto.spi.type.VarbinaryType.VARBINARY;
+
+public class SpatialPartitioningStateSerializer
+ implements AccumulatorStateSerializer
+{
+ @Override
+ public Type getSerializedType()
+ {
+ // TODO: make serializer optional in case of non decomposable aggregation
+ return VARBINARY;
+ }
+
+ @Override
+ public void serialize(SpatialPartitioningState state, BlockBuilder out)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void deserialize(Block block, int index, SpatialPartitioningState state)
+ {
+ throw new UnsupportedOperationException();
+ }
+}
diff --git a/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/aggregation/ConvexHullAggregation.java b/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/aggregation/ConvexHullAggregation.java
index 0270516c8de89..4e90d076885fb 100644
--- a/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/aggregation/ConvexHullAggregation.java
+++ b/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/aggregation/ConvexHullAggregation.java
@@ -28,15 +28,8 @@
import com.google.common.base.Joiner;
import io.airlift.slice.Slice;
-import java.util.EnumSet;
import java.util.Set;
-import static com.facebook.presto.geospatial.GeometryType.LINE_STRING;
-import static com.facebook.presto.geospatial.GeometryType.MULTI_LINE_STRING;
-import static com.facebook.presto.geospatial.GeometryType.MULTI_POINT;
-import static com.facebook.presto.geospatial.GeometryType.MULTI_POLYGON;
-import static com.facebook.presto.geospatial.GeometryType.POINT;
-import static com.facebook.presto.geospatial.GeometryType.POLYGON;
import static com.facebook.presto.plugin.geospatial.GeometryType.GEOMETRY;
import static com.facebook.presto.plugin.geospatial.GeometryType.GEOMETRY_TYPE_NAME;
import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT;
@@ -51,6 +44,7 @@
public class ConvexHullAggregation
{
private static final Joiner OR_JOINER = Joiner.on(" or ");
+
private ConvexHullAggregation() {}
@InputFunction
@@ -58,8 +52,6 @@ public static void input(@AggregationState GeometryState state,
@SqlType(GEOMETRY_TYPE_NAME) Slice input)
{
OGCGeometry geometry = GeometrySerde.deserialize(input);
- // There is a bug when the input is of GEOMETRY_COLLECTION. see https://github.com/Esri/geometry-api-java/issues/194
- validateType("convex_hull_agg", geometry, EnumSet.of(POINT, MULTI_POINT, LINE_STRING, MULTI_LINE_STRING, POLYGON, MULTI_POLYGON));
if (state.getGeometry() == null) {
state.setGeometry(geometry.convexHull());
}
diff --git a/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/aggregation/GeometryUnionAgg.java b/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/aggregation/GeometryUnionAgg.java
index 61bdf9086020c..6370100543457 100644
--- a/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/aggregation/GeometryUnionAgg.java
+++ b/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/aggregation/GeometryUnionAgg.java
@@ -29,8 +29,8 @@
import static com.facebook.presto.plugin.geospatial.GeometryType.GEOMETRY_TYPE_NAME;
/**
- * Aggregate form of ST_Union which takes a set of geometries and unions them into a single geometry, resulting in no intersecting
- * regions. The output may be a multi-geometry, a single geometry or a geometry collection.
+ * Aggregate form of ST_Union which takes a set of geometries and unions them into a single geometry using an iterative approach,
+ * resulting in no intersecting regions. The output may be a multi-geometry, a single geometry or a geometry collection.
*/
@Description("Returns a geometry that represents the point set union of the input geometries.")
@AggregationFunction("geometry_union_agg")
diff --git a/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/BenchmarkGeometryAggregations.java b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/BenchmarkGeometryAggregations.java
new file mode 100644
index 0000000000000..4539122606fa4
--- /dev/null
+++ b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/BenchmarkGeometryAggregations.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.facebook.presto.plugin.geospatial;
+
+import com.facebook.presto.plugin.memory.MemoryConnectorFactory;
+import com.facebook.presto.testing.LocalQueryRunner;
+import com.facebook.presto.testing.MaterializedResult;
+import com.google.common.collect.ImmutableMap;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+import org.openjdk.jmh.runner.options.VerboseMode;
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.stream.Collectors;
+
+import static com.facebook.presto.testing.TestingSession.testSessionBuilder;
+import static java.lang.String.format;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static org.openjdk.jmh.annotations.Mode.AverageTime;
+import static org.openjdk.jmh.annotations.Scope.Thread;
+
+@OutputTimeUnit(MILLISECONDS)
+@BenchmarkMode(AverageTime)
+@Fork(3)
+@Warmup(iterations = 10)
+@Measurement(iterations = 10)
+public class BenchmarkGeometryAggregations
+{
+ @State(Thread)
+ public static class Context
+ {
+ private LocalQueryRunner queryRunner;
+
+ public LocalQueryRunner getQueryRunner()
+ {
+ return queryRunner;
+ }
+
+ @Setup
+ public void setUp()
+ throws IOException
+ {
+ queryRunner = new LocalQueryRunner(testSessionBuilder()
+ .setCatalog("memory")
+ .setSchema("default")
+ .build());
+ queryRunner.installPlugin(new GeoPlugin());
+ queryRunner.createCatalog("memory", new MemoryConnectorFactory(), ImmutableMap.of());
+
+ Path path = Paths.get(BenchmarkGeometryAggregations.class.getClassLoader().getResource("us-states.tsv").getPath());
+ String polygonValues = Files.lines(path)
+ .map(line -> line.split("\t"))
+ .map(parts -> format("('%s', '%s')", parts[0], parts[1]))
+ .collect(Collectors.joining(","));
+
+ queryRunner.execute(
+ format("CREATE TABLE memory.default.us_states AS SELECT ST_GeometryFromText(t.wkt) AS geom FROM (VALUES %s) as t (name, wkt)",
+ polygonValues));
+ }
+
+ @TearDown
+ public void tearDown()
+ {
+ queryRunner.close();
+ queryRunner = null;
+ }
+ }
+
+ @Benchmark
+ public MaterializedResult benchmarkArrayUnion(Context context)
+ {
+ return context.getQueryRunner()
+ .execute("SELECT geometry_union(array_agg(p.geom)) FROM us_states p");
+ }
+
+ @Benchmark
+ public MaterializedResult benchmarkUnion(Context context)
+ {
+ return context.getQueryRunner()
+ .execute("SELECT geometry_union_agg(p.geom) FROM us_states p");
+ }
+
+ @Benchmark
+ public MaterializedResult benchmarkConvexHull(Context context)
+ {
+ return context.getQueryRunner()
+ .execute("SELECT convex_hull_agg(p.geom) FROM us_states p");
+ }
+
+ @Test
+ public void verify()
+ throws IOException
+ {
+ Context context = new Context();
+ try {
+ context.setUp();
+
+ BenchmarkGeometryAggregations benchmark = new BenchmarkGeometryAggregations();
+ benchmark.benchmarkUnion(context);
+ benchmark.benchmarkArrayUnion(context);
+ benchmark.benchmarkConvexHull(context);
+ }
+ finally {
+ context.queryRunner.close();
+ }
+ }
+
+ public static void main(String[] args)
+ throws Exception
+ {
+ new BenchmarkGeometryAggregations().verify();
+
+ Options options = new OptionsBuilder()
+ .verbosity(VerboseMode.NORMAL)
+ .include(".*" + BenchmarkGeometryAggregations.class.getSimpleName() + ".*")
+ .build();
+
+ new Runner(options).run();
+ }
+}
diff --git a/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestBingTileFunctions.java b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestBingTileFunctions.java
index d56e032f48ff6..61bd55955100d 100644
--- a/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestBingTileFunctions.java
+++ b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestBingTileFunctions.java
@@ -130,7 +130,7 @@ private void assertBingTilesAroundWithRadius(
double longitude,
int zoomLevel,
double radius,
- String...expectedQuadKeys)
+ String... expectedQuadKeys)
{
assertFunction(
format("transform(bing_tiles_around(%s, %s, %s, %s), x -> bing_tile_quadkey(x))",
@@ -239,7 +239,7 @@ public void testBingTilesWithRadiusBadInput()
// Too many tiles
assertInvalidFunction("bing_tiles_around(30.12, 60.0, 20, 100)",
- "The number of input tiles is too large (more than 1M) to compute a set of covering Bing tiles.");
+ "The number of tiles covering input rectangle exceeds the limit of 1M. Number of tiles: 36699364. Radius: 100.0 km. Zoom level: 20.");
}
@Test
@@ -454,7 +454,8 @@ public void testGeometryToBingTiles()
assertInvalidFunction("geometry_to_bing_tiles(ST_Point(60, 30.12), 40)", "Zoom level must be <= 23");
// Input rectangle too large
- assertInvalidFunction("geometry_to_bing_tiles(ST_Envelope(ST_GeometryFromText('LINESTRING (0 0, 80 80)')), 16)", "The number of input tiles is too large (more than 1M) to compute a set of covering Bing tiles.");
+ assertInvalidFunction("geometry_to_bing_tiles(ST_Envelope(ST_GeometryFromText('LINESTRING (0 0, 80 80)')), 16)",
+ "The number of tiles covering input rectangle exceeds the limit of 1M. Number of tiles: 370085804. Rectangle: xMin=0.00, yMin=0.00, xMax=80.00, yMax=80.00. Zoom level: 16.");
assertFunction("cardinality(geometry_to_bing_tiles(ST_Envelope(ST_GeometryFromText('LINESTRING (0 0, 80 80)')), 5))", BIGINT, 104L);
// Input polygon too complex
diff --git a/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestTransformSpatialPredicateToJoin.java b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestExtractSpatialInnerJoin.java
similarity index 98%
rename from presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestTransformSpatialPredicateToJoin.java
rename to presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestExtractSpatialInnerJoin.java
index ea8e54975bc4f..a6cd9f8389a96 100644
--- a/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestTransformSpatialPredicateToJoin.java
+++ b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestExtractSpatialInnerJoin.java
@@ -13,7 +13,7 @@
*/
package com.facebook.presto.plugin.geospatial;
-import com.facebook.presto.sql.planner.iterative.rule.TransformSpatialPredicates.TransformSpatialPredicateToJoin;
+import com.facebook.presto.sql.planner.iterative.rule.ExtractSpatialJoins.ExtractSpatialInnerJoin;
import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest;
import com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder;
import com.facebook.presto.sql.planner.iterative.rule.test.RuleAssert;
@@ -28,10 +28,10 @@
import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.values;
import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER;
-public class TestTransformSpatialPredicateToJoin
+public class TestExtractSpatialInnerJoin
extends BaseRuleTest
{
- public TestTransformSpatialPredicateToJoin()
+ public TestExtractSpatialInnerJoin()
{
super(new GeoPlugin());
}
@@ -352,6 +352,6 @@ public void testPushDownAnd()
private RuleAssert assertRuleApplication()
{
- return tester().assertThat(new TransformSpatialPredicateToJoin(tester().getMetadata()));
+ return tester().assertThat(new ExtractSpatialInnerJoin(tester().getMetadata(), tester().getSplitManager(), tester().getPageSourceManager()));
}
}
diff --git a/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestTransformSpatialPredicateToLeftJoin.java b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestExtractSpatialLeftJoin.java
similarity index 82%
rename from presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestTransformSpatialPredicateToLeftJoin.java
rename to presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestExtractSpatialLeftJoin.java
index c28e17aff5626..656e6ffdeaae9 100644
--- a/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestTransformSpatialPredicateToLeftJoin.java
+++ b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestExtractSpatialLeftJoin.java
@@ -14,7 +14,7 @@
package com.facebook.presto.plugin.geospatial;
import com.facebook.presto.sql.planner.assertions.PlanMatchPattern;
-import com.facebook.presto.sql.planner.iterative.rule.TransformSpatialPredicates.TransformSpatialPredicateToLeftJoin;
+import com.facebook.presto.sql.planner.iterative.rule.ExtractSpatialJoins.ExtractSpatialLeftJoin;
import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest;
import com.facebook.presto.sql.planner.iterative.rule.test.RuleAssert;
import com.google.common.collect.ImmutableMap;
@@ -28,10 +28,10 @@
import static com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder.expression;
import static com.facebook.presto.sql.planner.plan.JoinNode.Type.LEFT;
-public class TestTransformSpatialPredicateToLeftJoin
+public class TestExtractSpatialLeftJoin
extends BaseRuleTest
{
- public TestTransformSpatialPredicateToLeftJoin()
+ public TestExtractSpatialLeftJoin()
{
super(new GeoPlugin());
}
@@ -48,15 +48,6 @@ public void testDoesNotFire()
expression("ST_Contains(ST_GeometryFromText('POLYGON ...'), b)")))
.doesNotFire();
- // symbols
- assertRuleApplication()
- .on(p ->
- p.join(LEFT,
- p.values(p.symbol("a")),
- p.values(p.symbol("b")),
- expression("ST_Contains(a, b)")))
- .doesNotFire();
-
// OR operand
assertRuleApplication()
.on(p ->
@@ -85,6 +76,46 @@ public void testDoesNotFire()
.doesNotFire();
}
+ @Test
+ public void testConvertToSpatialJoin()
+ {
+ // symbols
+ assertRuleApplication()
+ .on(p ->
+ p.join(LEFT,
+ p.values(p.symbol("a")),
+ p.values(p.symbol("b")),
+ p.expression("ST_Contains(a, b)")))
+ .matches(
+ spatialLeftJoin("ST_Contains(a, b)",
+ values(ImmutableMap.of("a", 0)),
+ values(ImmutableMap.of("b", 0))));
+
+ // AND
+ assertRuleApplication()
+ .on(p ->
+ p.join(LEFT,
+ p.values(p.symbol("a"), p.symbol("name_1")),
+ p.values(p.symbol("b"), p.symbol("name_2")),
+ p.expression("name_1 != name_2 AND ST_Contains(a, b)")))
+ .matches(
+ spatialLeftJoin("name_1 != name_2 AND ST_Contains(a, b)",
+ values(ImmutableMap.of("a", 0, "name_1", 1)),
+ values(ImmutableMap.of("b", 0, "name_2", 1))));
+
+ // AND
+ assertRuleApplication()
+ .on(p ->
+ p.join(LEFT,
+ p.values(p.symbol("a1"), p.symbol("a2")),
+ p.values(p.symbol("b1"), p.symbol("b2")),
+ p.expression("ST_Contains(a1, b1) AND ST_Contains(a2, b2)")))
+ .matches(
+ spatialLeftJoin("ST_Contains(a1, b1) AND ST_Contains(a2, b2)",
+ values(ImmutableMap.of("a1", 0, "a2", 1)),
+ values(ImmutableMap.of("b1", 0, "b2", 1))));
+ }
+
@Test
public void testPushDownFirstArgument()
{
@@ -190,6 +221,6 @@ public void testPushDownAnd()
private RuleAssert assertRuleApplication()
{
- return tester().assertThat(new TransformSpatialPredicateToLeftJoin(tester().getMetadata()));
+ return tester().assertThat(new ExtractSpatialLeftJoin(tester().getMetadata(), tester().getSplitManager(), tester().getPageSourceManager()));
}
}
diff --git a/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestGeoFunctions.java b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestGeoFunctions.java
index 907545680e9eb..5fca6d28dbe13 100644
--- a/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestGeoFunctions.java
+++ b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestGeoFunctions.java
@@ -15,18 +15,21 @@
import com.esri.core.geometry.Point;
import com.esri.core.geometry.ogc.OGCPoint;
+import com.facebook.presto.geospatial.KdbTreeUtils;
+import com.facebook.presto.geospatial.Rectangle;
import com.facebook.presto.operator.scalar.AbstractTestFunctions;
import com.facebook.presto.spi.block.Block;
import com.facebook.presto.spi.block.BlockBuilder;
import com.facebook.presto.spi.type.ArrayType;
-import com.facebook.presto.spi.type.Type;
import com.google.common.collect.ImmutableList;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
+import java.util.Arrays;
import java.util.List;
+import java.util.stream.Collectors;
-import static com.facebook.presto.metadata.FunctionExtractor.extractFunctions;
+import static com.facebook.presto.geospatial.KdbTree.buildKdbTree;
import static com.facebook.presto.plugin.geospatial.GeometryType.GEOMETRY;
import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT;
import static com.facebook.presto.spi.type.BigintType.BIGINT;
@@ -35,8 +38,8 @@
import static com.facebook.presto.spi.type.IntegerType.INTEGER;
import static com.facebook.presto.spi.type.TinyintType.TINYINT;
import static com.facebook.presto.spi.type.VarcharType.VARCHAR;
-import static org.testng.Assert.assertEquals;
import static java.lang.String.format;
+import static org.testng.Assert.assertEquals;
public class TestGeoFunctions
extends AbstractTestFunctions
@@ -45,10 +48,64 @@ public class TestGeoFunctions
protected void registerFunctions()
{
GeoPlugin plugin = new GeoPlugin();
- for (Type type : plugin.getTypes()) {
- functionAssertions.getTypeRegistry().addType(type);
+ registerTypes(plugin);
+ registerFunctions(plugin);
+ }
+
+ @Test
+ public void testSpatialPartitions()
+ {
+ String kdbTreeJson = makeKdbTreeJson();
+
+ assertSpatialPartitions(kdbTreeJson, "POINT EMPTY", null);
+ // points inside partitions
+ assertSpatialPartitions(kdbTreeJson, "POINT (0 0)", ImmutableList.of(0));
+ assertSpatialPartitions(kdbTreeJson, "POINT (3 1)", ImmutableList.of(2));
+ // point on the border between two partitions
+ assertSpatialPartitions(kdbTreeJson, "POINT (1 2.5)", ImmutableList.of(1));
+ // point at the corner of three partitions
+ assertSpatialPartitions(kdbTreeJson, "POINT (4.5 2.5)", ImmutableList.of(4));
+ // points outside
+ assertSpatialPartitions(kdbTreeJson, "POINT (2 6)", ImmutableList.of());
+ assertSpatialPartitions(kdbTreeJson, "POINT (3 -1)", ImmutableList.of());
+ assertSpatialPartitions(kdbTreeJson, "POINT (10 3)", ImmutableList.of());
+
+ // geometry within a partition
+ assertSpatialPartitions(kdbTreeJson, "MULTIPOINT (5 0.1, 6 2)", ImmutableList.of(3));
+ // geometries spanning multiple partitions
+ assertSpatialPartitions(kdbTreeJson, "MULTIPOINT (5 0.1, 5.5 3, 6 2)", ImmutableList.of(3, 4));
+ assertSpatialPartitions(kdbTreeJson, "MULTIPOINT (3 2, 8 3)", ImmutableList.of(2, 3, 4, 5));
+ // geometry outside
+ assertSpatialPartitions(kdbTreeJson, "MULTIPOINT (2 6, 3 7)", ImmutableList.of());
+
+ // with distance
+ assertSpatialPartitions(kdbTreeJson, "POINT EMPTY", 1.2, null);
+ assertSpatialPartitions(kdbTreeJson, "POINT (1 1)", 1.2, ImmutableList.of(0));
+ assertSpatialPartitions(kdbTreeJson, "POINT (1 1)", 2.3, ImmutableList.of(0, 1, 2));
+ assertSpatialPartitions(kdbTreeJson, "MULTIPOINT (5 0.1, 6 2)", 0.2, ImmutableList.of(3));
+ assertSpatialPartitions(kdbTreeJson, "MULTIPOINT (5 0.1, 6 2)", 1.2, ImmutableList.of(2, 3, 4));
+ assertSpatialPartitions(kdbTreeJson, "MULTIPOINT (2 6, 3 7)", 1.2, ImmutableList.of());
+ }
+
+ private static String makeKdbTreeJson()
+ {
+ ImmutableList.Builder rectangles = ImmutableList.builder();
+ for (double x = 0; x < 10; x += 1) {
+ for (double y = 0; y < 5; y += 1) {
+ rectangles.add(new Rectangle(x, y, x + 1, y + 2));
+ }
}
- functionAssertions.getMetadata().addFunctions(extractFunctions(plugin.getFunctions()));
+ return KdbTreeUtils.toJson(buildKdbTree(10, new Rectangle(0, 0, 9, 4), rectangles.build()));
+ }
+
+ private void assertSpatialPartitions(String kdbTreeJson, String wkt, List expectedPartitions)
+ {
+ assertFunction(format("spatial_partitions(cast('%s' as KdbTree), ST_GeometryFromText('%s'))", kdbTreeJson, wkt), new ArrayType(INTEGER), expectedPartitions);
+ }
+
+ private void assertSpatialPartitions(String kdbTreeJson, String wkt, double distance, List expectedPartitions)
+ {
+ assertFunction(format("spatial_partitions(cast('%s' as KdbTree), ST_GeometryFromText('%s'), %s)", kdbTreeJson, wkt, distance), new ArrayType(INTEGER), expectedPartitions);
}
@Test
@@ -162,9 +219,9 @@ public void testSTConvexHull()
assertConvexHull("MULTILINESTRING EMPTY", "MULTILINESTRING EMPTY");
assertConvexHull("POLYGON EMPTY", "POLYGON EMPTY");
assertConvexHull("MULTIPOLYGON EMPTY", "MULTIPOLYGON EMPTY");
- assertConvexHullInvalidFunction("GEOMETRYCOLLECTION EMPTY", "ST_ConvexHull only applies to POINT or MULTI_POINT or LINE_STRING or MULTI_LINE_STRING or POLYGON or MULTI_POLYGON. Input type is: GEOMETRY_COLLECTION");
- assertConvexHullInvalidFunction("GEOMETRYCOLLECTION (POINT (1 1), POINT EMPTY)", "ST_ConvexHull only applies to POINT or MULTI_POINT or LINE_STRING or MULTI_LINE_STRING or POLYGON or MULTI_POLYGON. Input type is: GEOMETRY_COLLECTION");
- assertConvexHullInvalidFunction("GEOMETRYCOLLECTION (GEOMETRYCOLLECTION (POINT (1 1), GEOMETRYCOLLECTION (POINT (1 5), POINT (4 5), GEOMETRYCOLLECTION (POINT (3 4), POINT EMPTY))))", "ST_ConvexHull only applies to POINT or MULTI_POINT or LINE_STRING or MULTI_LINE_STRING or POLYGON or MULTI_POLYGON. Input type is: GEOMETRY_COLLECTION");
+ assertConvexHull("GEOMETRYCOLLECTION EMPTY", "GEOMETRYCOLLECTION EMPTY");
+ assertConvexHull("GEOMETRYCOLLECTION (POINT (1 1), POINT EMPTY)", "POINT (1 1)");
+ assertConvexHull("GEOMETRYCOLLECTION (GEOMETRYCOLLECTION (POINT (1 1), GEOMETRYCOLLECTION (POINT (1 5), POINT (4 5), GEOMETRYCOLLECTION (POINT (3 4), POINT EMPTY))))", "POLYGON ((1 1, 4 5, 1 5, 1 1))");
// test single geometry
assertConvexHull("POINT (1 1)", "POINT (1 1)");
@@ -182,15 +239,15 @@ public void testSTConvexHull()
assertConvexHull("LINESTRING (20 20, 30 30)", "LINESTRING (20 20, 30 30)");
assertConvexHull("MULTILINESTRING ((0 0, 3 3), (1 1, 2 2), (2 2, 4 4), (5 5, 8 8))", "LINESTRING (0 0, 8 8)");
assertConvexHull("MULTIPOINT (0 1, 1 2, 2 3, 3 4, 4 5, 5 6)", "LINESTRING (0 1, 5 6)");
- assertConvexHullInvalidFunction("GEOMETRYCOLLECTION (POINT (0 0), LINESTRING (1 1, 4 4, 2 2), POINT (10 10), POLYGON ((5 5, 7 7)), POINT (2 2), LINESTRING (6 6, 9 9), POLYGON ((1 1)))", "ST_ConvexHull only applies to POINT or MULTI_POINT or LINE_STRING or MULTI_LINE_STRING or POLYGON or MULTI_POLYGON. Input type is: GEOMETRY_COLLECTION");
- assertConvexHullInvalidFunction("GEOMETRYCOLLECTION (GEOMETRYCOLLECTION (POINT (2 2), POINT (1 1)), POINT (3 3))", "ST_ConvexHull only applies to POINT or MULTI_POINT or LINE_STRING or MULTI_LINE_STRING or POLYGON or MULTI_POLYGON. Input type is: GEOMETRY_COLLECTION");
+ assertConvexHull("GEOMETRYCOLLECTION (POINT (0 0), LINESTRING (1 1, 4 4, 2 2), POINT (10 10), POLYGON ((5 5, 7 7)), POINT (2 2), LINESTRING (6 6, 9 9), POLYGON ((1 1)))", "LINESTRING (0 0, 10 10)");
+ assertConvexHull("GEOMETRYCOLLECTION (GEOMETRYCOLLECTION (POINT (2 2), POINT (1 1)), POINT (3 3))", "LINESTRING (3 3, 1 1)");
// not all points are on the same line
assertConvexHull("MULTILINESTRING ((1 1, 5 1, 6 6), (2 4, 4 0), (2 -4, 4 4), (3 -2, 4 -3))", "POLYGON ((1 1, 2 -4, 4 -3, 5 1, 6 6, 2 4, 1 1))");
assertConvexHull("MULTIPOINT (0 2, 1 0, 3 0, 4 0, 4 2, 2 2, 2 4)", "POLYGON ((0 2, 1 0, 4 0, 4 2, 2 4, 0 2))");
assertConvexHull("MULTIPOLYGON (((0 3, 2 0, 3 6), (2 1, 2 3, 5 3, 5 1), (1 7, 2 4, 4 2, 5 6, 3 8)))", "POLYGON ((0 3, 2 0, 5 1, 5 6, 3 8, 1 7, 0 3))");
- assertConvexHullInvalidFunction("GEOMETRYCOLLECTION (POINT (2 3), LINESTRING (2 8, 7 10), POINT (8 10), POLYGON ((4 4, 4 8, 9 8, 6 6, 6 4, 8 3, 6 1)), POINT (4 2), LINESTRING (3 6, 5 5), POLYGON ((7 5, 7 6, 8 6, 8 5)))", "ST_ConvexHull only applies to POINT or MULTI_POINT or LINE_STRING or MULTI_LINE_STRING or POLYGON or MULTI_POLYGON. Input type is: GEOMETRY_COLLECTION");
- assertConvexHullInvalidFunction("GEOMETRYCOLLECTION (GEOMETRYCOLLECTION (POINT (2 3), LINESTRING (2 8, 7 10), GEOMETRYCOLLECTION (POINT (8 10))), POLYGON ((4 4, 4 8, 9 8, 6 6, 6 4, 8 3, 6 1)), POINT (4 2), LINESTRING (3 6, 5 5), POLYGON ((7 5, 7 6, 8 6, 8 5)))", "ST_ConvexHull only applies to POINT or MULTI_POINT or LINE_STRING or MULTI_LINE_STRING or POLYGON or MULTI_POLYGON. Input type is: GEOMETRY_COLLECTION");
+ assertConvexHull("GEOMETRYCOLLECTION (POINT (2 3), LINESTRING (2 8, 7 10), POINT (8 10), POLYGON ((4 4, 4 8, 9 8, 6 6, 6 4, 8 3, 6 1)), POINT (4 2), LINESTRING (3 6, 5 5), POLYGON ((7 5, 7 6, 8 6, 8 5)))", "POLYGON ((2 3, 6 1, 8 3, 9 8, 8 10, 7 10, 2 8, 2 3))");
+ assertConvexHull("GEOMETRYCOLLECTION (GEOMETRYCOLLECTION (POINT (2 3), LINESTRING (2 8, 7 10), GEOMETRYCOLLECTION (POINT (8 10))), POLYGON ((4 4, 4 8, 9 8, 6 6, 6 4, 8 3, 6 1)), POINT (4 2), LINESTRING (3 6, 5 5), POLYGON ((7 5, 7 6, 8 6, 8 5)))", "POLYGON ((2 3, 6 1, 8 3, 9 8, 8 10, 7 10, 2 8, 2 3))");
// single-element multi-geometries and geometry collections
assertConvexHull("MULTILINESTRING ((1 1, 5 1, 6 6))", "POLYGON ((1 1, 5 1, 6 6, 1 1))");
@@ -198,11 +255,11 @@ public void testSTConvexHull()
assertConvexHull("MULTIPOINT (0 2)", "POINT (0 2)");
assertConvexHull("MULTIPOLYGON (((0 3, 2 0, 3 6)))", "POLYGON ((0 3, 2 0, 3 6, 0 3))");
assertConvexHull("MULTIPOLYGON (((0 0, 4 0, 4 4, 0 4, 2 2)))", "POLYGON ((0 0, 4 0, 4 4, 0 4, 0 0))");
- assertConvexHullInvalidFunction("GEOMETRYCOLLECTION (POINT (2 3))", "ST_ConvexHull only applies to POINT or MULTI_POINT or LINE_STRING or MULTI_LINE_STRING or POLYGON or MULTI_POLYGON. Input type is: GEOMETRY_COLLECTION");
- assertConvexHullInvalidFunction("GEOMETRYCOLLECTION (LINESTRING (1 1, 5 1, 6 6))", "ST_ConvexHull only applies to POINT or MULTI_POINT or LINE_STRING or MULTI_LINE_STRING or POLYGON or MULTI_POLYGON. Input type is: GEOMETRY_COLLECTION");
- assertConvexHullInvalidFunction("GEOMETRYCOLLECTION (LINESTRING (1 1, 5 1, 1 4, 5 4))", "ST_ConvexHull only applies to POINT or MULTI_POINT or LINE_STRING or MULTI_LINE_STRING or POLYGON or MULTI_POLYGON. Input type is: GEOMETRY_COLLECTION");
- assertConvexHullInvalidFunction("GEOMETRYCOLLECTION (POLYGON ((0 3, 2 0, 3 6)))", "ST_ConvexHull only applies to POINT or MULTI_POINT or LINE_STRING or MULTI_LINE_STRING or POLYGON or MULTI_POLYGON. Input type is: GEOMETRY_COLLECTION");
- assertConvexHullInvalidFunction("GEOMETRYCOLLECTION (POLYGON ((0 0, 4 0, 4 4, 0 4, 2 2)))", "ST_ConvexHull only applies to POINT or MULTI_POINT or LINE_STRING or MULTI_LINE_STRING or POLYGON or MULTI_POLYGON. Input type is: GEOMETRY_COLLECTION");
+ assertConvexHull("GEOMETRYCOLLECTION (POINT (2 3))", "POINT (2 3)");
+ assertConvexHull("GEOMETRYCOLLECTION (LINESTRING (1 1, 5 1, 6 6))", "POLYGON ((1 1, 5 1, 6 6, 1 1))");
+ assertConvexHull("GEOMETRYCOLLECTION (LINESTRING (1 1, 5 1, 1 4, 5 4))", "POLYGON ((1 1, 5 1, 5 4, 1 4, 1 1))");
+ assertConvexHull("GEOMETRYCOLLECTION (POLYGON ((0 3, 2 0, 3 6)))", "POLYGON ((0 3, 2 0, 3 6, 0 3))");
+ assertConvexHull("GEOMETRYCOLLECTION (POLYGON ((0 0, 4 0, 4 4, 0 4, 2 2)))", "POLYGON ((0 0, 4 0, 4 4, 0 4, 0 0))");
}
private void assertConvexHull(String inputWKT, String expectWKT)
@@ -210,11 +267,6 @@ private void assertConvexHull(String inputWKT, String expectWKT)
assertFunction(format("ST_AsText(ST_ConvexHull(ST_GeometryFromText('%s')))", inputWKT), VARCHAR, expectWKT);
}
- private void assertConvexHullInvalidFunction(String inputWKT, String errorMessage)
- {
- assertInvalidFunction(format("ST_ConvexHull(ST_GeometryFromText('%s'))", inputWKT), errorMessage);
- }
-
@Test
public void testSTCoordDim()
{
@@ -543,6 +595,15 @@ public void testSTDistance()
assertFunction("ST_Distance(ST_GeometryFromText('MULTILINESTRING ((1 1, 5 1), (2 4, 4 4))'), ST_GeometryFromText('LINESTRING (10 20, 20 50)'))", DOUBLE, 17.08800749063506);
assertFunction("ST_Distance(ST_GeometryFromText('POLYGON ((1 1, 1 3, 3 3, 3 1))'), ST_GeometryFromText('POLYGON ((4 4, 4 5, 5 5, 5 4))'))", DOUBLE, 1.4142135623730951);
assertFunction("ST_Distance(ST_GeometryFromText('MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1)), ((0 0, 0 2, 2 2, 2 0)))'), ST_GeometryFromText('POLYGON ((10 100, 30 10))'))", DOUBLE, 27.892651361962706);
+
+ assertFunction("ST_Distance(ST_GeometryFromText('POINT EMPTY'), ST_Point(150, 150))", DOUBLE, null);
+ assertFunction("ST_Distance(ST_Point(50, 100), ST_GeometryFromText('POINT EMPTY'))", DOUBLE, null);
+ assertFunction("ST_Distance(ST_GeometryFromText('POINT EMPTY'), ST_GeometryFromText('POINT EMPTY'))", DOUBLE, null);
+ assertFunction("ST_Distance(ST_GeometryFromText('MULTIPOINT EMPTY'), ST_GeometryFromText('Point (50 100)'))", DOUBLE, null);
+ assertFunction("ST_Distance(ST_GeometryFromText('LINESTRING (50 100, 50 200)'), ST_GeometryFromText('LINESTRING EMPTY'))", DOUBLE, null);
+ assertFunction("ST_Distance(ST_GeometryFromText('MULTILINESTRING EMPTY'), ST_GeometryFromText('LINESTRING (10 20, 20 50)'))", DOUBLE, null);
+ assertFunction("ST_Distance(ST_GeometryFromText('POLYGON ((1 1, 1 3, 3 3, 3 1))'), ST_GeometryFromText('POLYGON EMPTY'))", DOUBLE, null);
+ assertFunction("ST_Distance(ST_GeometryFromText('MULTIPOLYGON EMPTY'), ST_GeometryFromText('POLYGON ((10 100, 30 10))'))", DOUBLE, null);
}
@Test
@@ -811,7 +872,8 @@ public void testSTUnion()
"LINESTRING EMPTY",
"MULTILINESTRING EMPTY",
"POLYGON EMPTY",
- "MULTIPOLYGON EMPTY");
+ "MULTIPOLYGON EMPTY",
+ "GEOMETRYCOLLECTION EMPTY");
List simpleWkts =
ImmutableList.of(
"POINT (1 2)",
@@ -819,12 +881,8 @@ public void testSTUnion()
"LINESTRING (0 0, 2 2, 4 4)",
"MULTILINESTRING ((0 0, 2 2, 4 4), (5 5, 7 7, 9 9))",
"POLYGON ((0 0, 1 0, 1 1, 0 1, 0 0))",
- "MULTIPOLYGON (((1 1, 3 1, 3 3, 1 3, 1 1)), ((2 4, 6 4, 6 6, 2 6, 2 4)))");
-
- // invalid type GEOMETRYCOLLECTION
- for (String simpleWkt : simpleWkts) {
- assertInvalidGeometryCollectionUnion(simpleWkt);
- }
+ "MULTIPOLYGON (((1 1, 3 1, 3 3, 1 3, 1 1)), ((2 4, 6 4, 6 6, 2 6, 2 4)))",
+ "GEOMETRYCOLLECTION (LINESTRING (0 5, 5 5), POLYGON ((1 1, 3 1, 3 3, 1 3, 1 1)))");
// empty geometry
for (String emptyWkt : emptyWkts) {
@@ -845,19 +903,22 @@ public void testSTUnion()
assertUnion("MULTILINESTRING ((0 0, 2 2, 4 4), (5 5, 7 7, 9 9))", "MULTILINESTRING ((5 5, 7 7, 9 9), (11 11, 13 13, 15 15))", "MULTILINESTRING ((0 0, 2 2, 4 4), (5 5, 7 7, 9 9), (11 11, 13 13, 15 15))");
assertUnion("POLYGON ((0 0, 1 0, 1 1, 0 1, 0 0))", "POLYGON ((1 0, 2 0, 2 1, 1 1, 1 0))", "POLYGON ((0 0, 1 0, 2 0, 2 1, 1 1, 0 1, 0 0))");
assertUnion("MULTIPOLYGON (((0 0, 1 0, 1 1, 0 1, 0 0)))", "MULTIPOLYGON (((1 0, 2 0, 2 1, 1 1, 1 0)))", "POLYGON ((0 0, 1 0, 2 0, 2 1, 1 1, 0 1, 0 0))");
+ assertUnion("GEOMETRYCOLLECTION (POLYGON ((0 0, 1 0, 1 1, 0 1, 0 0)), POINT (1 2))", "GEOMETRYCOLLECTION (POLYGON ((1 0, 2 0, 2 1, 1 1, 1 0)), MULTIPOINT ((1 2), (3 4)))", "GEOMETRYCOLLECTION (MULTIPOINT ((1 2), (3 4)), POLYGON ((0 0, 1 0, 2 0, 2 1, 1 1, 0 1, 0 0)))");
// within union
assertUnion("MULTIPOINT ((20 20), (25 25))", "POINT (25 25)", "MULTIPOINT ((20 20), (25 25))");
assertUnion("LINESTRING (20 20, 30 30)", "POINT (25 25)", "LINESTRING (20 20, 25 25, 30 30)");
assertUnion("LINESTRING (20 20, 30 30)", "LINESTRING (25 25, 27 27)", "LINESTRING (20 20, 25 25, 27 27, 30 30)");
assertUnion("POLYGON ((0 0, 4 0, 4 4, 0 4, 0 0))", "POLYGON ((1 1, 1 2, 2 2, 2 1, 1 1))", "POLYGON ((0 0, 4 0, 4 4, 0 4, 0 0))");
- assertUnion("MULTIPOLYGON (((0 0 , 0 2, 2 2, 2 0)), ((2 2, 2 4, 4 4, 4 2)))", "POLYGON ((2 2, 2 3, 3 3, 3 2))", "MULTIPOLYGON (((0 0, 2 0, 2 2, 0 2, 0 0)), ((2 2, 3 2, 4 2, 4 4, 2 4, 2 3, 2 2)))");
+ assertUnion("MULTIPOLYGON (((0 0 , 0 2, 2 2, 2 0)), ((2 2, 2 4, 4 4, 4 2)))", "POLYGON ((2 2, 2 3, 3 3, 3 2))", "MULTIPOLYGON (((2 2, 3 2, 4 2, 4 4, 2 4, 2 3, 2 2)), ((0 0, 2 0, 2 2, 0 2, 0 0)))");
+ assertUnion("GEOMETRYCOLLECTION (POLYGON ((0 0, 4 0, 4 4, 0 4, 0 0)), MULTIPOINT ((20 20), (25 25)))", "GEOMETRYCOLLECTION (POLYGON ((1 1, 1 2, 2 2, 2 1, 1 1)), POINT (25 25))", "GEOMETRYCOLLECTION (MULTIPOINT ((20 20), (25 25)), POLYGON ((0 0, 4 0, 4 4, 0 4, 0 0)))");
// overlap union
assertUnion("LINESTRING (1 1, 3 1)", "LINESTRING (2 1, 4 1)", "LINESTRING (1 1, 2 1, 3 1, 4 1)");
assertUnion("MULTILINESTRING ((1 1, 3 1))", "MULTILINESTRING ((2 1, 4 1))", "LINESTRING (1 1, 2 1, 3 1, 4 1)");
assertUnion("POLYGON ((1 1, 3 1, 3 3, 1 3, 1 1))", "POLYGON ((2 2, 4 2, 4 4, 2 4, 2 2))", "POLYGON ((1 1, 3 1, 3 2, 4 2, 4 4, 2 4, 2 3, 1 3, 1 1))");
assertUnion("MULTIPOLYGON (((1 1, 3 1, 3 3, 1 3, 1 1)))", "MULTIPOLYGON (((2 2, 4 2, 4 4, 2 4, 2 2)))", "POLYGON ((1 1, 3 1, 3 2, 4 2, 4 4, 2 4, 2 3, 1 3, 1 1))");
+ assertUnion("GEOMETRYCOLLECTION (POLYGON ((1 1, 3 1, 3 3, 1 3, 1 1)), LINESTRING (1 1, 3 1))", "GEOMETRYCOLLECTION (POLYGON ((2 2, 4 2, 4 4, 2 4, 2 2)), LINESTRING (2 1, 4 1))", "GEOMETRYCOLLECTION (LINESTRING (3 1, 4 1), POLYGON ((1 1, 2 1, 3 1, 3 2, 4 2, 4 4, 2 4, 2 3, 1 3, 1 1)))");
}
private void assertUnion(String leftWkt, String rightWkt, String expectWkt)
@@ -923,6 +984,99 @@ private void assertSTGeometryN(String wkt, int index, String expected)
assertFunction("ST_ASText(ST_GeometryN(ST_GeometryFromText('" + wkt + "')," + index + "))", VARCHAR, expected);
}
+ @Test
+ public void testSTLineString()
+ {
+ // General case, 2+ points
+ assertFunction("ST_LineString(array[ST_Point(1,2), ST_Point(3,4)])", GEOMETRY, "LINESTRING (1 2, 3 4)");
+ assertFunction("ST_LineString(array[ST_Point(1,2), ST_Point(3,4), ST_Point(5, 6)])", GEOMETRY, "LINESTRING (1 2, 3 4, 5 6)");
+ assertFunction("ST_LineString(array[ST_Point(1,2), ST_Point(3,4), ST_Point(5,6), ST_Point(7,8)])", GEOMETRY, "LINESTRING (1 2, 3 4, 5 6, 7 8)");
+
+ // Other ways of creating points
+ assertFunction("ST_LineString(array[ST_GeometryFromText('POINT (1 2)'), ST_GeometryFromText('POINT (3 4)')])", GEOMETRY, "LINESTRING (1 2, 3 4)");
+
+ // Duplicate consecutive points throws exception
+ assertInvalidFunction("ST_LineString(array[ST_Point(1, 2), ST_Point(1, 2)])", INVALID_FUNCTION_ARGUMENT, "Invalid input to ST_LineString: consecutive duplicate points at index 2");
+ assertFunction("ST_LineString(array[ST_Point(1, 2), ST_Point(3, 4), ST_Point(1, 2)])", GEOMETRY, "LINESTRING (1 2, 3 4, 1 2)");
+
+ // Single point
+ assertFunction("ST_LineString(array[ST_Point(9,10)])", GEOMETRY, "LINESTRING EMPTY");
+
+ // Zero points
+ assertFunction("ST_LineString(array[])", GEOMETRY, "LINESTRING EMPTY");
+
+ // Only points can be passed
+ assertInvalidFunction("ST_LineString(array[ST_Point(7,8), ST_GeometryFromText('LINESTRING (1 2, 3 4)')])", INVALID_FUNCTION_ARGUMENT, "ST_LineString takes only an array of valid points, LineString was passed");
+
+ // Nulls points are invalid
+ assertInvalidFunction("ST_LineString(array[NULL])", INVALID_FUNCTION_ARGUMENT, "Invalid input to ST_LineString: null point at index 1");
+ assertInvalidFunction("ST_LineString(array[ST_Point(1,2), NULL])", INVALID_FUNCTION_ARGUMENT, "Invalid input to ST_LineString: null point at index 2");
+ assertInvalidFunction("ST_LineString(array[ST_Point(1, 2), NULL, ST_Point(3, 4)])", INVALID_FUNCTION_ARGUMENT, "Invalid input to ST_LineString: null point at index 2");
+ assertInvalidFunction("ST_LineString(array[ST_Point(1, 2), NULL, ST_Point(3, 4), NULL])", INVALID_FUNCTION_ARGUMENT, "Invalid input to ST_LineString: null point at index 2");
+
+ // Empty points are invalid
+ assertInvalidFunction("ST_LineString(array[ST_GeometryFromText('POINT EMPTY')])", INVALID_FUNCTION_ARGUMENT, "Invalid input to ST_LineString: empty point at index 1");
+ assertInvalidFunction("ST_LineString(array[ST_Point(1,2), ST_GeometryFromText('POINT EMPTY')])", INVALID_FUNCTION_ARGUMENT, "Invalid input to ST_LineString: empty point at index 2");
+ assertInvalidFunction("ST_LineString(array[ST_Point(1,2), ST_GeometryFromText('POINT EMPTY'), ST_Point(3,4)])", INVALID_FUNCTION_ARGUMENT, "Invalid input to ST_LineString: empty point at index 2");
+ assertInvalidFunction("ST_LineString(array[ST_Point(1,2), ST_GeometryFromText('POINT EMPTY'), ST_Point(3,4), ST_GeometryFromText('POINT EMPTY')])", INVALID_FUNCTION_ARGUMENT, "Invalid input to ST_LineString: empty point at index 2");
+ }
+
+ @Test
+ public void testMultiPoint()
+ {
+ // General case, 2+ points
+ assertMultiPoint("MULTIPOINT ((1 2), (3 4))", "POINT (1 2)", "POINT (3 4)");
+ assertMultiPoint("MULTIPOINT ((1 2), (3 4), (5 6))", "POINT (1 2)", "POINT (3 4)", "POINT (5 6)");
+ assertMultiPoint("MULTIPOINT ((1 2), (3 4), (5 6), (7 8))", "POINT (1 2)", "POINT (3 4)", "POINT (5 6)", "POINT (7 8)");
+
+ // Duplicate points work
+ assertMultiPoint("MULTIPOINT ((1 2), (1 2))", "POINT (1 2)", "POINT (1 2)");
+ assertMultiPoint("MULTIPOINT ((1 2), (3 4), (1 2))", "POINT (1 2)", "POINT (3 4)", "POINT (1 2)");
+
+ // Single point
+ assertMultiPoint("MULTIPOINT ((1 2))", "POINT (1 2)");
+
+ // Empty array
+ assertFunction("ST_MultiPoint(array[])", GEOMETRY, null);
+
+ // Only points can be passed
+ assertInvalidMultiPoint("geometry is not a point: LineString at index 2", "POINT (7 8)", "LINESTRING (1 2, 3 4)");
+
+ // Null point raises exception
+ assertInvalidFunction("ST_MultiPoint(array[null])", "Invalid input to ST_MultiPoint: null at index 1");
+ assertInvalidMultiPoint("null at index 3", "POINT (1 2)", "POINT (1 2)", null);
+ assertInvalidMultiPoint("null at index 2", "POINT (1 2)", null, "POINT (3 4)");
+ assertInvalidMultiPoint("null at index 2", "POINT (1 2)", null, "POINT (3 4)", null);
+
+ // Empty point raises exception
+ assertInvalidMultiPoint("empty point at index 1", "POINT EMPTY");
+ assertInvalidMultiPoint("empty point at index 2", "POINT (1 2)", "POINT EMPTY");
+ }
+
+ private void assertMultiPoint(String expectedWkt, String... pointWkts)
+ {
+ assertFunction(
+ format(
+ "ST_MultiPoint(array[%s])",
+ Arrays.stream(pointWkts)
+ .map(wkt -> wkt == null ? "null" : format("ST_GeometryFromText('%s')", wkt))
+ .collect(Collectors.joining(","))),
+ GEOMETRY,
+ expectedWkt);
+ }
+
+ private void assertInvalidMultiPoint(String errorMessage, String... pointWkts)
+ {
+ assertInvalidFunction(
+ format(
+ "ST_MultiPoint(array[%s])",
+ Arrays.stream(pointWkts)
+ .map(wkt -> wkt == null ? "null" : format("ST_GeometryFromText('%s')", wkt))
+ .collect(Collectors.joining(","))),
+ INVALID_FUNCTION_ARGUMENT,
+ format("Invalid input to ST_MultiPoint: %s", errorMessage));
+ }
+
@Test
public void testSTPointN()
{
@@ -970,6 +1124,7 @@ private void assertSTGeometries(String wkt, String... expected)
assertFunction(String.format("transform(ST_Geometries(ST_GeometryFromText('%s')), x -> ST_ASText(x))", wkt), new ArrayType(VARCHAR), ImmutableList.copyOf(expected));
}
+ @Test
public void testSTInteriorRingN()
{
assertInvalidInteriorRingN("POINT EMPTY", 0, "POINT");
@@ -997,6 +1152,7 @@ private void assertInvalidInteriorRingN(String wkt, int index, String geometryTy
assertInvalidFunction(format("ST_InteriorRingN(ST_GeometryFromText('%s'), %d)", wkt, index), format("ST_InteriorRingN only applies to POLYGON. Input type is: %s", geometryType));
}
+ @Test
public void testSTGeometryType()
{
assertFunction("ST_GeometryType(ST_Point(1, 4))", VARCHAR, "ST_Point");
@@ -1008,4 +1164,44 @@ public void testSTGeometryType()
assertFunction("ST_GeometryType(ST_GeometryFromText('GEOMETRYCOLLECTION(POINT(4 6),LINESTRING(4 6, 7 10))'))", VARCHAR, "ST_GeomCollection");
assertFunction("ST_GeometryType(ST_Envelope(ST_GeometryFromText('LINESTRING (1 1, 2 2)')))", VARCHAR, "ST_Polygon");
}
+
+ @Test
+ public void testSTGeometryFromBinary()
+ {
+ assertFunction("ST_GeomFromBinary(null)", GEOMETRY, null);
+
+ // empty geometries
+ assertGeomFromBinary("POINT EMPTY");
+ assertGeomFromBinary("MULTIPOINT EMPTY");
+ assertGeomFromBinary("LINESTRING EMPTY");
+ assertGeomFromBinary("MULTILINESTRING EMPTY");
+ assertGeomFromBinary("POLYGON EMPTY");
+ assertGeomFromBinary("MULTIPOLYGON EMPTY");
+ assertGeomFromBinary("GEOMETRYCOLLECTION EMPTY");
+
+ // valid nonempty geometries
+ assertGeomFromBinary("POINT (1 2)");
+ assertGeomFromBinary("MULTIPOINT ((1 2), (3 4))");
+ assertGeomFromBinary("LINESTRING (0 0, 1 2, 3 4)");
+ assertGeomFromBinary("MULTILINESTRING ((1 1, 5 1), (2 4, 4 4))");
+ assertGeomFromBinary("POLYGON ((0 0, 1 0, 1 1, 0 1, 0 0))");
+ assertGeomFromBinary("POLYGON ((0 0, 3 0, 3 3, 0 3, 0 0), (1 1, 1 2, 2 2, 2 1, 1 1))");
+ assertGeomFromBinary("MULTIPOLYGON (((1 1, 3 1, 3 3, 1 3, 1 1)), ((2 4, 6 4, 6 6, 2 6, 2 4)))");
+ assertGeomFromBinary("GEOMETRYCOLLECTION (POINT (1 2), LINESTRING (0 0, 1 2, 3 4), POLYGON ((0 0, 1 0, 1 1, 0 1, 0 0)))");
+
+ // array of geometries
+ assertFunction("transform(array[ST_AsBinary(ST_Point(1, 2)), ST_AsBinary(ST_Point(3, 4))], wkb -> ST_AsText(ST_GeomFromBinary(wkb)))", new ArrayType(VARCHAR), ImmutableList.of("POINT (1 2)", "POINT (3 4)"));
+
+ // invalid geometries
+ assertGeomFromBinary("MULTIPOINT ((0 0), (0 1), (1 1), (0 1))");
+ assertGeomFromBinary("LINESTRING (0 0, 0 1, 0 1, 1 1, 1 0, 0 0)");
+
+ // invalid binary
+ assertInvalidFunction("ST_GeomFromBinary(from_hex('deadbeef'))", "Invalid WKB");
+ }
+
+ private void assertGeomFromBinary(String wkt)
+ {
+ assertFunction(format("ST_AsText(ST_GeomFromBinary(ST_AsBinary(ST_GeometryFromText('%s'))))", wkt), VARCHAR, wkt);
+ }
}
diff --git a/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestKdbTreeCasts.java b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestKdbTreeCasts.java
new file mode 100644
index 0000000000000..916e3016edd51
--- /dev/null
+++ b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestKdbTreeCasts.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.facebook.presto.plugin.geospatial;
+
+import com.facebook.presto.geospatial.KdbTreeUtils;
+import com.facebook.presto.geospatial.Rectangle;
+import com.facebook.presto.operator.scalar.AbstractTestFunctions;
+import com.google.common.collect.ImmutableList;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import static com.facebook.presto.geospatial.KdbTree.buildKdbTree;
+import static com.facebook.presto.spi.type.VarcharType.VARCHAR;
+import static java.lang.String.format;
+
+public class TestKdbTreeCasts
+ extends AbstractTestFunctions
+{
+ @BeforeClass
+ protected void registerFunctions()
+ {
+ GeoPlugin plugin = new GeoPlugin();
+ registerTypes(plugin);
+ registerFunctions(plugin);
+ }
+
+ @Test
+ public void test()
+ {
+ String kdbTreeJson = makeKdbTreeJson();
+ assertFunction(format("typeof(cast('%s' AS KdbTree))", kdbTreeJson), VARCHAR, "KdbTree");
+ assertFunction(format("typeof(cast('%s' AS KDBTree))", kdbTreeJson), VARCHAR, "KdbTree");
+ assertFunction(format("typeof(cast('%s' AS kdbTree))", kdbTreeJson), VARCHAR, "KdbTree");
+ assertFunction(format("typeof(cast('%s' AS kdbtree))", kdbTreeJson), VARCHAR, "KdbTree");
+
+ assertInvalidCast("typeof(cast('' AS KdbTree))", "Invalid JSON string for KDB tree");
+ }
+
+ private String makeKdbTreeJson()
+ {
+ ImmutableList.Builder rectangles = ImmutableList.builder();
+ for (double x = 0; x < 10; x += 1) {
+ for (double y = 0; y < 5; y += 1) {
+ rectangles.add(new Rectangle(x, y, x + 1, y + 2));
+ }
+ }
+ return KdbTreeUtils.toJson(buildKdbTree(100, new Rectangle(0, 0, 9, 4), rectangles.build()));
+ }
+}
diff --git a/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestRewriteSpatialPartitioningAggregation.java b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestRewriteSpatialPartitioningAggregation.java
new file mode 100644
index 0000000000000..189d4ee7e7ca6
--- /dev/null
+++ b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestRewriteSpatialPartitioningAggregation.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.facebook.presto.plugin.geospatial;
+
+import com.facebook.presto.sql.planner.iterative.rule.RewriteSpatialPartitioningAggregation;
+import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest;
+import com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder;
+import com.facebook.presto.sql.planner.iterative.rule.test.RuleAssert;
+import com.facebook.presto.sql.planner.plan.AggregationNode;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.testng.annotations.Test;
+
+import static com.facebook.presto.plugin.geospatial.GeometryType.GEOMETRY;
+import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.aggregation;
+import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.expression;
+import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.functionCall;
+import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.project;
+import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.values;
+
+public class TestRewriteSpatialPartitioningAggregation
+ extends BaseRuleTest
+{
+ public TestRewriteSpatialPartitioningAggregation()
+ {
+ super(new GeoPlugin());
+ }
+
+ @Test
+ public void testDoesNotFire()
+ {
+ assertRuleApplication()
+ .on(p -> p.aggregation(a ->
+ a.globalGrouping()
+ .step(AggregationNode.Step.FINAL)
+ .addAggregation(p.symbol("sp"), PlanBuilder.expression("spatial_partitioning(geometry, 10)"), ImmutableList.of(GEOMETRY))
+ .source(p.values(p.symbol("geometry")))))
+ .doesNotFire();
+ }
+
+ @Test
+ public void test()
+ {
+ assertRuleApplication()
+ .on(p -> p.aggregation(a ->
+ a.globalGrouping()
+ .step(AggregationNode.Step.FINAL)
+ .addAggregation(p.symbol("sp"), PlanBuilder.expression("spatial_partitioning(geometry)"), ImmutableList.of(GEOMETRY))
+ .source(p.values(p.symbol("geometry")))))
+ .matches(
+ aggregation(
+ ImmutableMap.of("sp", functionCall("spatial_partitioning", ImmutableList.of("envelope", "partition_count"))),
+ project(
+ ImmutableMap.of("partition_count", expression("100"),
+ "envelope", expression("ST_Envelope(geometry)")),
+ values("geometry"))));
+
+ assertRuleApplication()
+ .on(p -> p.aggregation(a ->
+ a.globalGrouping()
+ .step(AggregationNode.Step.FINAL)
+ .addAggregation(p.symbol("sp"), PlanBuilder.expression("spatial_partitioning(ST_Envelope(geometry))"), ImmutableList.of(GEOMETRY))
+ .source(p.values(p.symbol("geometry")))))
+ .matches(
+ aggregation(
+ ImmutableMap.of("sp", functionCall("spatial_partitioning", ImmutableList.of("envelope", "partition_count"))),
+ project(
+ ImmutableMap.of("partition_count", expression("100"),
+ "envelope", expression("ST_Envelope(geometry)")),
+ values("geometry"))));
+ }
+
+ private RuleAssert assertRuleApplication()
+ {
+ return tester().assertThat(new RewriteSpatialPartitioningAggregation(tester().getMetadata()));
+ }
+}
diff --git a/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestSpatialJoinOperator.java b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestSpatialJoinOperator.java
index 9e45698839eaf..309c8160aacb3 100644
--- a/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestSpatialJoinOperator.java
+++ b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestSpatialJoinOperator.java
@@ -14,6 +14,9 @@
package com.facebook.presto.plugin.geospatial;
import com.facebook.presto.RowPagesBuilder;
+import com.facebook.presto.geospatial.KdbTree;
+import com.facebook.presto.geospatial.KdbTreeUtils;
+import com.facebook.presto.geospatial.Rectangle;
import com.facebook.presto.operator.Driver;
import com.facebook.presto.operator.DriverContext;
import com.facebook.presto.operator.InternalJoinFilterFunction;
@@ -31,8 +34,8 @@
import com.facebook.presto.spi.Page;
import com.facebook.presto.spi.PrestoException;
import com.facebook.presto.sql.gen.JoinFilterFunctionCompiler;
-import com.facebook.presto.sql.planner.plan.JoinNode.Type;
import com.facebook.presto.sql.planner.plan.PlanNodeId;
+import com.facebook.presto.sql.planner.plan.SpatialJoinNode.Type;
import com.facebook.presto.testing.MaterializedResult;
import com.facebook.presto.testing.TestingTaskContext;
import com.google.common.collect.ImmutableList;
@@ -54,14 +57,17 @@
import static com.facebook.presto.RowPagesBuilder.rowPagesBuilder;
import static com.facebook.presto.SessionTestUtils.TEST_SESSION;
+import static com.facebook.presto.geospatial.KdbTree.Node.newInternal;
+import static com.facebook.presto.geospatial.KdbTree.Node.newLeaf;
import static com.facebook.presto.operator.OperatorAssertion.assertOperatorEquals;
import static com.facebook.presto.plugin.geospatial.GeoFunctions.stGeometryFromText;
import static com.facebook.presto.plugin.geospatial.GeoFunctions.stPoint;
import static com.facebook.presto.plugin.geospatial.GeometryType.GEOMETRY;
import static com.facebook.presto.spi.type.DoubleType.DOUBLE;
+import static com.facebook.presto.spi.type.IntegerType.INTEGER;
import static com.facebook.presto.spi.type.VarcharType.VARCHAR;
-import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER;
-import static com.facebook.presto.sql.planner.plan.JoinNode.Type.LEFT;
+import static com.facebook.presto.sql.planner.plan.SpatialJoinNode.Type.INNER;
+import static com.facebook.presto.sql.planner.plan.SpatialJoinNode.Type.LEFT;
import static com.facebook.presto.testing.MaterializedResult.resultBuilder;
import static io.airlift.concurrent.Threads.daemonThreadsNamed;
import static java.util.concurrent.Executors.newScheduledThreadPool;
@@ -74,6 +80,13 @@
@Test(singleThreaded = true)
public class TestSpatialJoinOperator
{
+ private static final String KDB_TREE_JSON = KdbTreeUtils.toJson(
+ new KdbTree(newInternal(new Rectangle(-2, -2, 15, 15),
+ newInternal(new Rectangle(-2, -2, 6, 15),
+ newLeaf(new Rectangle(-2, -2, 6, 1), 1),
+ newLeaf(new Rectangle(-2, 1, 6, 15), 2)),
+ newLeaf(new Rectangle(6, -2, 15, 15), 0))));
+
// 2 intersecting polygons: A and B
private static final Slice POLYGON_A = stGeometryFromText(Slices.utf8Slice("POLYGON ((0 0, -0.5 2.5, 0 5, 2.5 5.5, 5 5, 5.5 2.5, 5 0, 2.5 -0.5, 0 0))"));
private static final Slice POLYGON_B = stGeometryFromText(Slices.utf8Slice("POLYGON ((4 4, 3.5 7, 4 10, 7 10.5, 10 10, 10.5 7, 10 4, 7 3.5, 4 4))"));
@@ -175,9 +188,9 @@ public void testSpatialLeftJoin()
private void assertSpatialJoin(TaskContext taskContext, Type joinType, RowPagesBuilder buildPages, RowPagesBuilder probePages, MaterializedResult expected)
{
- DriverContext driverContext = taskContext.addPipelineContext(0, true, true).addDriverContext();
+ DriverContext driverContext = taskContext.addPipelineContext(0, true, true, false).addDriverContext();
PagesSpatialIndexFactory pagesSpatialIndexFactory = buildIndex(driverContext, (build, probe, r) -> build.contains(probe), Optional.empty(), Optional.empty(), buildPages);
- OperatorFactory joinOperatorFactory = new SpatialJoinOperatorFactory(2, new PlanNodeId("test"), joinType, probePages.getTypes(), Ints.asList(1), 0, pagesSpatialIndexFactory);
+ OperatorFactory joinOperatorFactory = new SpatialJoinOperatorFactory(2, new PlanNodeId("test"), joinType, probePages.getTypes(), Ints.asList(1), 0, Optional.empty(), pagesSpatialIndexFactory);
assertOperatorEquals(joinOperatorFactory, driverContext, probePages.build(), expected);
}
@@ -251,7 +264,7 @@ public void testYield()
// verify we will yield #match times totally
TaskContext taskContext = createTaskContext();
- DriverContext driverContext = taskContext.addPipelineContext(0, true, true).addDriverContext();
+ DriverContext driverContext = taskContext.addPipelineContext(0, true, true, false).addDriverContext();
// force a yield for every match
AtomicInteger filterFunctionCalls = new AtomicInteger();
@@ -284,7 +297,7 @@ public void testYield()
}
List probeInput = probePages.build();
- OperatorFactory joinOperatorFactory = new SpatialJoinOperatorFactory(2, new PlanNodeId("test"), INNER, probePages.getTypes(), Ints.asList(1), 0, pagesSpatialIndexFactory);
+ OperatorFactory joinOperatorFactory = new SpatialJoinOperatorFactory(2, new PlanNodeId("test"), INNER, probePages.getTypes(), Ints.asList(1), 0, Optional.empty(), pagesSpatialIndexFactory);
Operator operator = joinOperatorFactory.createOperator(driverContext);
assertTrue(operator.needsInput());
@@ -311,7 +324,7 @@ public void testYield()
public void testDistanceQuery()
{
TaskContext taskContext = createTaskContext();
- DriverContext driverContext = taskContext.addPipelineContext(0, true, true).addDriverContext();
+ DriverContext driverContext = taskContext.addPipelineContext(0, true, true, false).addDriverContext();
RowPagesBuilder buildPages = rowPagesBuilder(ImmutableList.of(GEOMETRY, VARCHAR, DOUBLE))
.row(stPoint(0, 0), "0_0", 1.5)
@@ -331,7 +344,7 @@ public void testDistanceQuery()
.row(stPoint(3, 1), "3_1")
.pageBreak()
.row(stPoint(10, 1), "10_1");
- OperatorFactory joinOperatorFactory = new SpatialJoinOperatorFactory(2, new PlanNodeId("test"), INNER, probePages.getTypes(), Ints.asList(1), 0, pagesSpatialIndexFactory);
+ OperatorFactory joinOperatorFactory = new SpatialJoinOperatorFactory(2, new PlanNodeId("test"), INNER, probePages.getTypes(), Ints.asList(1), 0, Optional.empty(), pagesSpatialIndexFactory);
MaterializedResult expected = resultBuilder(taskContext.getSession(), ImmutableList.of(VARCHAR, VARCHAR))
.row("0_1", "0_0")
@@ -345,10 +358,74 @@ public void testDistanceQuery()
assertOperatorEquals(joinOperatorFactory, driverContext, probePages.build(), expected);
}
+ @Test
+ public void testDistributedSpatialJoin()
+ {
+ TaskContext taskContext = createTaskContext();
+ DriverContext driverContext = taskContext.addPipelineContext(0, true, true, true).addDriverContext();
+
+ RowPagesBuilder buildPages = rowPagesBuilder(ImmutableList.of(GEOMETRY, VARCHAR, INTEGER))
+ .row(POLYGON_A, "A", 1)
+ .row(POLYGON_A, "A", 2)
+ .row(null, "null", null)
+ .pageBreak()
+ .row(POLYGON_B, "B", 0)
+ .row(POLYGON_B, "B", 2);
+
+ RowPagesBuilder probePages = rowPagesBuilder(ImmutableList.of(GEOMETRY, VARCHAR, INTEGER))
+ .row(POINT_X, "x", 2)
+ .row(null, "null", null)
+ .row(POINT_Y, "y", 2)
+ .pageBreak()
+ .row(POINT_Z, "z", 0);
+
+ MaterializedResult expected = resultBuilder(taskContext.getSession(), ImmutableList.of(VARCHAR, VARCHAR))
+ .row("x", "A")
+ .row("y", "A")
+ .row("y", "B")
+ .row("z", "B")
+ .build();
+
+ PagesSpatialIndexFactory pagesSpatialIndexFactory = buildIndex(driverContext, (build, probe, r) -> build.contains(probe), Optional.empty(), Optional.of(2), Optional.of(KDB_TREE_JSON), Optional.empty(), buildPages);
+ OperatorFactory joinOperatorFactory = new SpatialJoinOperatorFactory(2, new PlanNodeId("test"), INNER, probePages.getTypes(), Ints.asList(1), 0, Optional.of(2), pagesSpatialIndexFactory);
+ assertOperatorEquals(joinOperatorFactory, driverContext, probePages.build(), expected);
+ }
+
+ @Test
+ public void testDistributedSpatialSelfJoin()
+ {
+ TaskContext taskContext = createTaskContext();
+ DriverContext driverContext = taskContext.addPipelineContext(0, true, true, true).addDriverContext();
+
+ RowPagesBuilder pages = rowPagesBuilder(ImmutableList.of(GEOMETRY, VARCHAR, INTEGER))
+ .row(POLYGON_A, "A", 1)
+ .row(POLYGON_A, "A", 2)
+ .row(null, "null", null)
+ .pageBreak()
+ .row(POLYGON_B, "B", 0)
+ .row(POLYGON_B, "B", 2);
+
+ MaterializedResult expected = resultBuilder(taskContext.getSession(), ImmutableList.of(VARCHAR, VARCHAR))
+ .row("A", "A")
+ .row("A", "B")
+ .row("B", "A")
+ .row("B", "B")
+ .build();
+
+ PagesSpatialIndexFactory pagesSpatialIndexFactory = buildIndex(driverContext, (build, probe, r) -> build.intersects(probe), Optional.empty(), Optional.of(2), Optional.of(KDB_TREE_JSON), Optional.empty(), pages);
+ OperatorFactory joinOperatorFactory = new SpatialJoinOperatorFactory(2, new PlanNodeId("test"), INNER, pages.getTypes(), Ints.asList(1), 0, Optional.of(2), pagesSpatialIndexFactory);
+ assertOperatorEquals(joinOperatorFactory, driverContext, pages.build(), expected);
+ }
+
private PagesSpatialIndexFactory buildIndex(DriverContext driverContext, SpatialPredicate spatialRelationshipTest, Optional radiusChannel, Optional filterFunction, RowPagesBuilder buildPages)
+ {
+ return buildIndex(driverContext, spatialRelationshipTest, radiusChannel, Optional.empty(), Optional.empty(), filterFunction, buildPages);
+ }
+
+ private PagesSpatialIndexFactory buildIndex(DriverContext driverContext, SpatialPredicate spatialRelationshipTest, Optional radiusChannel, Optional partitionChannel, Optional kdbTreeJson, Optional filterFunction, RowPagesBuilder buildPages)
{
Optional