[#9021] feat(lance-rest): Support create & list index for LRS

api/src/main/java/org/apache/gravitino/rel/TableChange.java

@@ -22,6 +22,7 @@
 
 import com.google.common.base.Preconditions;
 import java.util.Arrays;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 import org.apache.gravitino.annotation.Evolving;
@@ -453,6 +454,20 @@ static TableChange addIndex(IndexType type, String name, String[][] fieldNames)
     return new AddIndex(type, name, fieldNames);
   }
 
+  /**
+   * Create a TableChange for adding an index.
+   *
+   * @param type The type of the index.
+   * @param name The name of the index.
+   * @param fieldNames The field names of the index.
+   * @param properties The properties of the index.
+   * @return A TableChange for the add index.
+   */
+  static TableChange addIndex(
+      IndexType type, String name, String[][] fieldNames, Map<String, String> properties) {
+    return new AddIndex(type, name, fieldNames, properties);
+  }
+
   /**
    * Create a TableChange for deleting an index.
    *
@@ -747,15 +762,29 @@ final class AddIndex implements TableChange {
 
     private final String[][] fieldNames;
 
+    private final Map<String, String> properties;
+
     /**
      * @param type The type of the index.
      * @param name The name of the index.
      * @param fieldNames The field names of the index.
      */
     public AddIndex(IndexType type, String name, String[][] fieldNames) {
+      this(type, name, fieldNames, Map.of());
+    }
+
+    /**
+     * @param type The type of the index.
+     * @param name The name of the index.
+     * @param fieldNames The field names of the index.
+     * @param properties The properties of the index.
+     */
+    public AddIndex(
+        IndexType type, String name, String[][] fieldNames, Map<String, String> properties) {
       this.type = type;
       this.name = name;
       this.fieldNames = fieldNames;
+      this.properties = properties;
     }
 
     /**
@@ -779,20 +808,29 @@ public String[][] getFieldNames() {
       return fieldNames;
     }
 
+    /**
+     * @return The properties of the index.
+     */
+    public Map<String, String> getProperties() {
+      return properties;
+    }
+
     @Override
     public boolean equals(Object o) {
       if (this == o) return true;
       if (o == null || getClass() != o.getClass()) return false;
       AddIndex addIndex = (AddIndex) o;
       return type == addIndex.type
           && Objects.equals(name, addIndex.name)
-          && Arrays.deepEquals(fieldNames, addIndex.fieldNames);
+          && Arrays.deepEquals(fieldNames, addIndex.fieldNames)
+          && Objects.equals(properties, addIndex.properties);
     }
 
     @Override
     public int hashCode() {
       int result = Objects.hash(type, name);
       result = 31 * result + Arrays.hashCode(fieldNames);
+      result = 31 * result + Objects.hashCode(properties);
       return result;
     }
   }

api/src/main/java/org/apache/gravitino/rel/indexes/Index.java

@@ -19,6 +19,7 @@
 
 package org.apache.gravitino.rel.indexes;
 
+import java.util.Map;
 import org.apache.gravitino.annotation.Evolving;
 
 /**
@@ -44,6 +45,13 @@ public interface Index {
    */
   String[][] fieldNames();
 
+  /**
+   * @return The properties of the index.
+   */
+  default Map<String, String> properties() {
+    return Map.of();
+  }
+
   /**
    * The enum IndexType defines the type of the index. Currently, PRIMARY_KEY and UNIQUE_KEY are
    * supported.
@@ -102,15 +110,44 @@ enum IndexType {
      * Currently, this type is only applicable to Lance.
      */
     VECTOR,
-    /** IVF_FLAT (Inverted File with Flat quantization) is an indexing method used for efficient */
+
+    /**
+     * IVF_FLAT (Inverted File with Flat Quantization) is an indexing method used for efficient
+     * approximate nearest neighbor search in high-dimensional vector spaces. It stores the original
+     * vectors without any quantization.
+     */
     IVF_FLAT,
-    /** IVF_SQ (Inverted File with Scalar Quantization) is an indexing method used for efficient */
+    /**
+     * IVF_SQ (Inverted File with Scalar Quantization) is an indexing method used for efficient
+     * approximate nearest neighbor search in high-dimensional vector spaces. It applies scalar
+     * quantization to reduce the storage size of the vectors.
+     */
     IVF_SQ,
-    /** IVF_PQ (Inverted File with Product Quantization) is an indexing method used for efficient */
+    /**
+     * IVF_PQ (Inverted File with Product Quantization) is an indexing method used for efficient
+     * approximate nearest neighbor search in high-dimensional vector spaces. It applies product
+     * quantization to compress the vectors into smaller codes for faster search and reduced
+     * storage.
+     */
     IVF_PQ,
-    /** IVF_HNSW_FLAT */
+
+    /**
+     * IVF_HNSW_SQ is an indexing method that combines Inverted File (IVF) with Hierarchical
+     * Navigable Small World (HNSW) graphs and Scalar Quantization (SQ) for efficient approximate
+     * nearest neighbor search in high-dimensional vector spaces.
+     */
     IVF_HNSW_SQ,
-    /** IVF_HNSW_PQ */
-    IVF_HNSW_PQ;
+    /**
+     * IVF_HNSW_PQ is an indexing method that combines Inverted File (IVF) with Hierarchical
+     * Navigable Small World (HNSW) graphs and Product Quantization (PQ) for efficient approximate
+     * nearest neighbor search in high-dimensional vector spaces.
+     */
+    IVF_HNSW_PQ,
+    /**
+     * FTS index is a data structure used for efficient storage and retrieval of strings, enabling
+     * fast prefix-based searches and pattern matching. Currently, this type is only applicable to
+     * Lance.
+     */
+    FTS;
   }
 }

api/src/main/java/org/apache/gravitino/rel/indexes/Indexes.java

@@ -20,6 +20,7 @@
 
 import com.google.common.base.Objects;
 import java.util.Arrays;
+import java.util.Map;
 
 /** Helper methods to create index to pass into Apache Gravitino. */
 public class Indexes {
@@ -76,6 +77,28 @@ public static Index of(Index.IndexType indexType, String name, String[][] fieldN
         .build();
   }
 
+  /**
+   * Create an index.
+   *
+   * @param indexType The type of the index
+   * @param name The name of the index
+   * @param fieldNames The field names under the table contained in the index.
+   * @param properties The properties of the index.
+   * @return The index to be created.
+   */
+  public static Index of(
+      Index.IndexType indexType,
+      String name,
+      String[][] fieldNames,
+      Map<String, String> properties) {
+    return IndexImpl.builder()
+        .withIndexType(indexType)
+        .withName(name)
+        .withFieldNames(fieldNames)
+        .withProperties(properties)
+        .build();
+  }
+
   /** The user side implementation of the index. */
   public static final class IndexImpl implements Index {
     private final IndexType indexType;
@@ -84,17 +107,20 @@ public static final class IndexImpl implements Index {
 
     private final String[][] fieldNames;
 
+    private final Map<String, String> properties;
     /**
      * The constructor of the index.
      *
      * @param indexType The type of the index
      * @param name The name of the index
      * @param fieldNames The field names under the table contained in the index.
      */
-    private IndexImpl(IndexType indexType, String name, String[][] fieldNames) {
+    private IndexImpl(
+        IndexType indexType, String name, String[][] fieldNames, Map<String, String> properties) {
       this.indexType = indexType;
       this.name = name;
       this.fieldNames = fieldNames;
+      this.properties = properties;
     }
 
     /**
@@ -121,6 +147,11 @@ public String[][] fieldNames() {
       return fieldNames;
     }
 
+    @Override
+    public Map<String, String> properties() {
+      return properties;
+    }
+
     @Override
     public boolean equals(Object o) {
       if (this == o) {
@@ -132,12 +163,15 @@ public boolean equals(Object o) {
       IndexImpl index = (IndexImpl) o;
       return indexType == index.indexType
           && Objects.equal(name, index.name)
-          && Arrays.deepEquals(fieldNames, index.fieldNames);
+          && Arrays.deepEquals(fieldNames, index.fieldNames)
+          && Objects.equal(properties, index.properties);
     }
 
     @Override
     public int hashCode() {
-      return Objects.hashCode(indexType, name, Arrays.hashCode(fieldNames));
+      int result = Objects.hashCode(indexType, name, properties);
+      result = 31 * result + Arrays.deepHashCode(fieldNames);
+      return result;
     }
 
     /**
@@ -159,6 +193,9 @@ public static class Builder {
       /** The field names of the index. */
       protected String[][] fieldNames;
 
+      /** The properties of the index. */
+      protected Map<String, String> properties;
+
       /**
        * Set the type of the index.
        *
@@ -192,13 +229,24 @@ public Indexes.IndexImpl.Builder withFieldNames(String[][] fieldNames) {
         return this;
       }
 
+      /**
+       * Set the properties of the index.
+       *
+       * @param properties The properties of the index
+       * @return The builder for creating a new instance of IndexImpl.
+       */
+      public Indexes.IndexImpl.Builder withProperties(Map<String, String> properties) {
+        this.properties = properties;
+        return this;
+      }
+
       /**
        * Build a new instance of IndexImpl.
        *
        * @return The new instance.
        */
       public Index build() {
-        return new IndexImpl(indexType, name, fieldNames);
+        return new IndexImpl(indexType, name, fieldNames, properties);
       }
     }
   }

build.gradle.kts

@@ -858,17 +858,6 @@ tasks {
     destinationDirectory.set(projectDir.dir("distribution"))
   }
 
-  val assembleLanceRESTServer by registering(Tar::class) {
-    dependsOn("compileLanceRESTServer")
-    group = "gravitino distribution"
-    finalizedBy("checksumLanceRESTServerDistribution")
-    into("${rootProject.name}-lance-rest-server-$version-bin")
-    from(compileLanceRESTServer.map { it.outputs.files.single() })
-    compression = Compression.GZIP
-    archiveFileName.set("${rootProject.name}-lance-rest-server-$version-bin.tar.gz")
-    destinationDirectory.set(projectDir.dir("distribution"))
-  }
-
   val assembleIcebergRESTServer by registering(Tar::class) {
     dependsOn("compileIcebergRESTServer")
     group = "gravitino distribution"
@@ -880,6 +869,17 @@ tasks {
     destinationDirectory.set(projectDir.dir("distribution"))
   }
 
+  val assembleLanceRESTServer by registering(Tar::class) {
+    dependsOn("compileLanceRESTServer")
+    group = "gravitino distribution"
+    finalizedBy("checksumLanceRESTServerDistribution")
+    into("${rootProject.name}-lance-rest-server-$version-bin")
+    from(compileLanceRESTServer.map { it.outputs.files.single() })
+    compression = Compression.GZIP
+    archiveFileName.set("${rootProject.name}-lance-rest-server-$version-bin.tar.gz")
+    destinationDirectory.set(projectDir.dir("distribution"))
+  }
+
   register("checksumIcebergRESTServerDistribution") {
     group = "gravitino distribution"
     dependsOn(assembleIcebergRESTServer)
@@ -914,7 +914,12 @@ tasks {
 
   register("checksumDistribution") {
     group = "gravitino distribution"
-    dependsOn(assembleDistribution, "checksumTrinoConnector", "checksumIcebergRESTServerDistribution", "checksumLanceRESTServerDistribution")
+    dependsOn(
+      assembleDistribution,
+      "checksumTrinoConnector",
+      "checksumIcebergRESTServerDistribution",
+      "checksumLanceRESTServerDistribution"
+    )
     val archiveFile = assembleDistribution.flatMap { it.archiveFile }
     val checksumFile = archiveFile.map { archive ->
       archive.asFile.let { it.resolveSibling("${it.name}.sha256") }

catalogs/catalog-lakehouse-generic/build.gradle.kts

@@ -43,7 +43,11 @@ dependencies {
   implementation(libs.commons.io)
   implementation(libs.commons.lang3)
   implementation(libs.guava)
-  implementation(libs.lance)
+  implementation(libs.hadoop3.client.api)
+  implementation(libs.lance) {
+    exclude(group = "com.google.guava", module = "guava") // provided by gravitino
+    exclude(group = "org.apache.commons", module = "commons-lang3") // provided by gravitino
+  }
 
   annotationProcessor(libs.lombok)