PARQUET-116: Enforce that the filter object to be passed must be Seri…

…alizable
apache · Oct 28, 2014 · d5a2b9e · d5a2b9e
1 parent dfd0478
commit d5a2b9e
Show file tree

Hide file tree

Showing 9 changed files with 25 additions and 14 deletions.
diff --git a/parquet-column/src/main/java/parquet/filter2/predicate/FilterApi.java b/parquet-column/src/main/java/parquet/filter2/predicate/FilterApi.java
@@ -1,5 +1,7 @@
 package parquet.filter2.predicate;
 
+import java.io.Serializable;
+
 import parquet.common.schema.ColumnPath;
 import parquet.filter2.predicate.Operators.And;
 import parquet.filter2.predicate.Operators.BinaryColumn;
@@ -146,7 +148,7 @@ public static <T extends Comparable<T>, C extends Column<T> & SupportsLtGt> GtEq
    * Keeps records that pass the provided {@link UserDefinedPredicate}
    */
   public static <T extends Comparable<T>, U extends UserDefinedPredicate<T>>
-    UserDefined<T, U> userDefined(Column<T> column, Class<U> clazz, Object o) {
+    UserDefined<T, U> userDefined(Column<T> column, Class<U> clazz, Serializable o) {
     return new UserDefined<T, U>(column, clazz, o);
   }
 

diff --git a/parquet-column/src/main/java/parquet/filter2/predicate/Operators.java b/parquet-column/src/main/java/parquet/filter2/predicate/Operators.java
@@ -344,11 +344,11 @@ public static final class UserDefined<T extends Comparable<T>, U extends UserDef
     private final Column<T> column;
     private final Class<U> udpClass;
     private final String toString;
-    private final Object o;
+    private final Serializable o;
     private static final String INSTANTIATION_ERROR_MESSAGE =
         "Could not instantiate custom filter: %s. User defined predicates must be static classes with a default constructor.";
 
-    UserDefined(Column<T> column, Class<U> udpClass, Object o) {
+    UserDefined(Column<T> column, Class<U> udpClass, Serializable o) {
       this.column = checkNotNull(column, "column");
       this.udpClass = checkNotNull(udpClass, "udpClass");
       String name = getClass().getSimpleName().toLowerCase();
@@ -367,7 +367,7 @@ public Class<U> getUserDefinedPredicateClass() {
       return udpClass;
     }
 
-    public Object getFilterObject() {
+    public Serializable getFilterObject() {
       return o;
     }
 

diff --git a/parquet-column/src/main/java/parquet/filter2/predicate/UserDefinedPredicate.java b/parquet-column/src/main/java/parquet/filter2/predicate/UserDefinedPredicate.java
@@ -1,5 +1,7 @@
 package parquet.filter2.predicate;
 
+import java.io.Serializable;
+
 /**
  * A UserDefinedPredicate decides whether a record should be kept or dropped, first by
  * inspecting meta data about a group of records to see if the entire group can be dropped,
@@ -24,7 +26,7 @@ public UserDefinedPredicate() { }
    * Return true to keep the record with this value, false to drop it.
    * o is a filter object that can be used for filtering the value.
    */
-  public abstract boolean keep(T value, Object o);
+  public abstract boolean keep(T value, Serializable o);
 
   /**
    * Given information about a group of records (eg, the min and max value)

diff --git a/parquet-column/src/test/java/parquet/filter2/predicate/DummyUdp.java b/parquet-column/src/test/java/parquet/filter2/predicate/DummyUdp.java
@@ -1,9 +1,11 @@
 package parquet.filter2.predicate;
 
+import java.io.Serializable;
+
 public class DummyUdp extends UserDefinedPredicate<Integer> {
 
   @Override
-  public boolean keep(Integer value, Object o) {
+  public boolean keep(Integer value, Serializable o) {
     return false;
   }
 

diff --git a/parquet-column/src/test/java/parquet/filter2/predicate/TestSchemaCompatibilityValidator.java b/parquet-column/src/test/java/parquet/filter2/predicate/TestSchemaCompatibilityValidator.java
@@ -2,6 +2,7 @@
 
 import org.junit.Test;
 
+import java.io.Serializable;
 import parquet.filter2.predicate.Operators.BinaryColumn;
 import parquet.filter2.predicate.Operators.IntColumn;
 import parquet.filter2.predicate.Operators.LongColumn;
@@ -51,7 +52,7 @@ public class TestSchemaCompatibilityValidator {
 
   static class LongDummyUdp extends UserDefinedPredicate<Long> {
     @Override
-    public boolean keep(Long value, Object o) {
+    public boolean keep(Long value, Serializable o) {
       return false;
     }
 

diff --git a/...generator/src/main/java/parquet/filter2/IncrementallyUpdatedFilterPredicateGenerator.java b/...generator/src/main/java/parquet/filter2/IncrementallyUpdatedFilterPredicateGenerator.java
@@ -50,6 +50,7 @@ private TypeInfo(String className, String primitiveName, boolean useComparable,
   public void run() throws IOException {
     add("package parquet.filter2.recordlevel;\n" +
         "\n" +
+        "import java.io.Serializable;\n" +
         "import parquet.common.schema.ColumnPath;\n" +
         "import parquet.filter2.predicate.Operators.Eq;\n" +
         "import parquet.filter2.predicate.Operators.Gt;\n" +
@@ -222,7 +223,7 @@ private void addUdpBegin() throws IOException {
         "\n" +
         "    final U udp = pred.getUserDefinedPredicate();\n" +
         "\n" +
-        "    final Object o = pred.getFilterObject();\n" +
+        "    final Serializable o = pred.getFilterObject();\n" +
         "\n");
   }
 

diff --git a/parquet-hadoop/src/test/java/parquet/filter2/recordlevel/TestRecordLevelFilters.java b/parquet-hadoop/src/test/java/parquet/filter2/recordlevel/TestRecordLevelFilters.java
@@ -2,6 +2,7 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Iterator;
@@ -147,7 +148,7 @@ public boolean keep(User u) {
   public static class StartWithP extends UserDefinedPredicate<Binary> {
 
     @Override
-    public boolean keep(Binary value, Object o) {
+    public boolean keep(Binary value, Serializable o) {
       if (value == null) {
         return false;
       }
@@ -168,7 +169,7 @@ public boolean inverseCanDrop(Statistics<Binary> statistics) {
   public static class SetInFilter extends UserDefinedPredicate<Long> {
 
     @Override
-    public boolean keep(Long value, Object o) {
+    public boolean keep(Long value, Serializable o) {
       if (value == null) {
         return false;
       }
@@ -208,7 +209,7 @@ public boolean keep(User u) {
   public void testIdIn() throws Exception {
     LongColumn name = longColumn("id");
 
-    Set<Long> h = new HashSet<Long>() {{
+    HashSet<Long> h = new HashSet<Long>() {{
           add(20L); add(27L); add(28L);
     }}; 
     FilterPredicate pred = userDefined(name, SetInFilter.class, h);

diff --git a/parquet-hadoop/src/test/java/parquet/filter2/statisticslevel/TestStatisticsFilter.java b/parquet-hadoop/src/test/java/parquet/filter2/statisticslevel/TestStatisticsFilter.java
@@ -3,6 +3,7 @@
 import java.util.Arrays;
 import java.util.HashSet;
 import java.util.List;
+import java.io.Serializable;
 
 import org.junit.Test;
 
@@ -219,10 +220,10 @@ public void testOr() {
     assertFalse(canDrop(or(no, no), columnMetas));
   }
 
-  public static class SevensAndEightsUdp extends UserDefinedPredicate<Integer> {
+  public static class SevensAndEightsUdp extends UserDefinedPredicate<Integer, Serializable> {
 
     @Override
-    public boolean keep(Integer value, Object o) {
+    public boolean keep(Integer value, Serializable o) {
       throw new RuntimeException("this method should not be called");
     }
 

diff --git a/parquet-scala/src/test/scala/parquet/filter2/dsl/DslTest.scala b/parquet-scala/src/test/scala/parquet/filter2/dsl/DslTest.scala
@@ -1,6 +1,7 @@
 package parquet.filter2.dsl
 
 import java.lang.{Double => JDouble, Integer => JInt}
+import java.io.Serializable;
 
 import org.junit.runner.RunWith
 import org.scalatest.FlatSpec
@@ -9,7 +10,7 @@ import parquet.filter2.predicate.Operators.{Or, UserDefined, DoubleColumn => JDo
 import parquet.filter2.predicate.{FilterApi, Statistics, UserDefinedPredicate}
 
 class DummyFilter extends UserDefinedPredicate[JInt] {
-  override def keep(value: JInt, o: Object): Boolean = false
+  override def keep(value: JInt, o: Serializable): Boolean = false
 
   override def canDrop(statistics: Statistics[JInt]): Boolean = false