[#8839] feat(catalogs): Support add index for lance table

api/build.gradle.kts

@@ -26,6 +26,8 @@ dependencies {
   implementation(libs.commons.lang3)
   implementation(libs.commons.collections4)
   implementation(libs.guava)
+  implementation(libs.jackson.annotations)
+  implementation(libs.jackson.databind)
 
   testImplementation(libs.junit.jupiter.api)
   testImplementation(libs.junit.jupiter.params)

api/src/main/java/org/apache/gravitino/rel/GenericTable.java

@@ -0,0 +1,47 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing,
+ *  software distributed under the License is distributed on an
+ *  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ *  KIND, either express or implied.  See the License for the
+ *  specific language governing permissions and limitations
+ *  under the License.
+ */
+
+package org.apache.gravitino.rel;
+
+/** A generic table interface that extends the Table interface. */
+public interface GenericTable extends Table {
+
+  /**
+   * Formats the table as a string representation.
+   *
+   * @return the formatted string representation of the table
+   */
+  String format();
+
+  /**
+   * Gets the location of the table.
+   *
+   * @return the location of the table
+   */
+  String location();
+
+  /**
+   * Indicates whether the table is external.
+   *
+   * @return true if the table is external, false otherwise
+   */
+  default boolean external() {
+    return false;
+  }
+}

api/src/main/java/org/apache/gravitino/rel/indexes/Index.java

@@ -67,5 +67,50 @@ enum IndexType {
      * UNIQUE KEY helps in avoiding redundancy and ensuring data accuracy in the database.
      */
     UNIQUE_KEY,
+
+    // The following index types are specific to Lance, for more, please see: IndexType in LanceDB
+    /**
+     * SCALAR index is used to optimize searches on scalar data types such as integers, floats,
+     * strings, etc. Currently, this type is only applicable to Lance.
+     */
+    SCALAR,
+    /**
+     * BTREE index is a balanced tree data structure that maintains sorted data and allows for
+     * logarithmic time complexity for search, insert, and delete operations. Currently, this type
+     * is only applicable to Lance.
+     */
+    BTREE,
+    /**
+     * Bitmap index is a type of database index that uses bit arrays (bitmaps) to represent the
+     * presence or absence of values in a column, enabling efficient querying and data retrieval.
+     * Currently, this type is only applicable to Lance.
+     */
+    BITMAP,
+    /**
+     * LABEL_LIST index is used to optimize searches on columns containing lists of labels or tags.
+     * Currently, this type is only applicable to Lance.
+     */
+    LABEL_LIST,
+    /**
+     * INVERTED index is a data structure used to optimize full-text searches by mapping terms to
+     * their locations within a dataset, allowing for quick retrieval of documents containing
+     * specific words or phrases. Currently, this type is only applicable to Lance.
+     */
+    INVERTED,
+    /**
+     * VECTOR index is used to optimize similarity searches in high-dimensional vector spaces.
+     * Currently, this type is only applicable to Lance.
+     */
+    VECTOR,
+    /** IVF_FLAT (Inverted File with Flat quantization) is an indexing method used for efficient */
+    IVF_FLAT,
+    /** IVF_SQ (Inverted File with Scalar Quantization) is an indexing method used for efficient */
+    IVF_SQ,
+    /** IVF_PQ (Inverted File with Product Quantization) is an indexing method used for efficient */
+    IVF_PQ,
+    /** IVF_HNSW_FLAT */
+    IVF_HNSW_SQ,
+    /** IVF_HNSW_PQ */
+    IVF_HNSW_PQ;
   }
 }

api/src/main/java/org/apache/gravitino/rel/indexes/Indexes.java

@@ -18,6 +18,22 @@
  */
 package org.apache.gravitino.rel.indexes;
 
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.JsonSerializer;
+import com.fasterxml.jackson.databind.SerializerProvider;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import java.io.IOException;
+import java.util.List;
+import java.util.Locale;
+
 /** Helper methods to create index to pass into Apache Gravitino. */
 public class Indexes {
 
@@ -73,10 +89,81 @@ public static Index of(Index.IndexType indexType, String name, String[][] fieldN
         .build();
   }
 
+  /** Custom JSON serializer for Index objects. */
+  public static class IndexSerializer extends JsonSerializer<Index> {
+    @Override
+    public void serialize(Index value, JsonGenerator gen, SerializerProvider serializers)
+        throws IOException {
+      gen.writeStartObject();
+      gen.writeStringField("indexType", value.type().name().toUpperCase(Locale.ROOT));
+      if (null != value.name()) {
+        gen.writeStringField("name", value.name());
+      }
+      gen.writeFieldName("fieldNames");
+      gen.writeObject(value.fieldNames());
+      gen.writeEndObject();
+    }
+  }
+
+  /** Custom JSON deserializer for Index objects. */
+  public static class IndexDeserializer extends JsonDeserializer<Index> {
+
+    @Override
+    public Index deserialize(JsonParser p, DeserializationContext ctxt) throws IOException {
+      JsonNode node = p.getCodec().readTree(p);
+      Preconditions.checkArgument(
+          node != null && !node.isNull() && node.isObject(),
+          "Index must be a valid JSON object, but found: %s",
+          node);
+
+      IndexImpl.Builder builder = IndexImpl.builder();
+      Preconditions.checkArgument(
+          node.has("indexType"), "Cannot parse index from missing type: %s", node);
+      String indexType = getString("indexType", node);
+      builder.withIndexType(Index.IndexType.valueOf(indexType.toUpperCase(Locale.ROOT)));
+      if (node.has("name")) {
+        builder.withName(getString("name", node));
+      }
+      Preconditions.checkArgument(
+          node.has("fieldNames"), "Cannot parse index from missing field names: %s", node);
+      List<String[]> fieldNames = Lists.newArrayList();
+      node.get("fieldNames").forEach(field -> fieldNames.add(getStringArray((ArrayNode) field)));
+      builder.withFieldNames(fieldNames.toArray(new String[0][0]));
+      return builder.build();
+    }
+
+    private static String[] getStringArray(ArrayNode node) {
+      String[] array = new String[node.size()];
+      for (int i = 0; i < node.size(); i++) {
+        array[i] = node.get(i).asText();
+      }
+      return array;
+    }
+
+    private static String getString(String property, JsonNode node) {
+      Preconditions.checkArgument(node.has(property), "Cannot parse missing string: %s", property);
+      JsonNode pNode = node.get(property);
+      return convertToString(property, pNode);
+    }
+
+    private static String convertToString(String property, JsonNode pNode) {
+      Preconditions.checkArgument(
+          pNode != null && !pNode.isNull() && pNode.isTextual(),
+          "Cannot parse to a string value %s: %s",
+          property,
+          pNode);
+      return pNode.asText();
+    }
+  }
+
   /** The user side implementation of the index. */
+  @JsonSerialize(using = IndexSerializer.class)
+  @JsonDeserialize(using = IndexDeserializer.class)
   public static final class IndexImpl implements Index {
     private final IndexType indexType;
+
     private final String name;
+
     private final String[][] fieldNames;
 
     /**

api/src/test/java/org/apache/gravitino/rel/TestIndex.java

@@ -0,0 +1,57 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing,
+ *  software distributed under the License is distributed on an
+ *  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ *  KIND, either express or implied.  See the License for the
+ *  specific language governing permissions and limitations
+ *  under the License.
+ */
+
+package org.apache.gravitino.rel;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.MapperFeature;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.databind.cfg.EnumFeature;
+import com.fasterxml.jackson.databind.json.JsonMapper;
+import org.apache.gravitino.rel.indexes.Index;
+import org.apache.gravitino.rel.indexes.Indexes;
+import org.apache.gravitino.rel.indexes.Indexes.IndexImpl;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class TestIndex {
+
+  @Test
+  void testIndexSerialization() throws JsonProcessingException {
+    String[][] fields = {{"column1"}, {"column2", "subcolumn"}};
+    Index index = Indexes.unique("test_index", fields);
+
+    JsonMapper jsonMapper =
+        JsonMapper.builder()
+            .configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false)
+            .configure(EnumFeature.WRITE_ENUMS_TO_LOWERCASE, true)
+            .enable(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS)
+            .disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
+            .configure(DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL, true)
+            .build();
+
+    String json = jsonMapper.writeValueAsString(index);
+
+    Index deserializedIndex = jsonMapper.readValue(json, IndexImpl.class);
+    Assertions.assertEquals(index.type(), deserializedIndex.type());
+    Assertions.assertEquals(index.name(), deserializedIndex.name());
+    Assertions.assertArrayEquals(index.fieldNames(), deserializedIndex.fieldNames());
+  }
+}

catalogs/catalog-generic-lakehouse/build.gradle.kts

@@ -43,6 +43,7 @@ dependencies {
   implementation(libs.commons.lang3)
   implementation(libs.guava)
   implementation(libs.hadoop3.client.api)
+  implementation(libs.lance)
 
   annotationProcessor(libs.lombok)