feat(lance): add Lance REST server packaging

bin/gravitino-lance-rest-server.sh.template

@@ -0,0 +1,206 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#set -ex
+USAGE="-e Usage: bin/gravitino-lance-rest-server.sh [--config <conf-dir>]\n\t
+        {start|run|stop|restart|status}"
+
+if [[ "$1" == "--config" ]]; then
+  shift
+  conf_dir="$1"
+  if [[ ! -d "${conf_dir}" ]]; then
+    echo "ERROR : ${conf_dir} is not a directory"
+    echo ${USAGE}
+    exit 1
+  else
+    export GRAVITINO_CONF_DIR="${conf_dir}"
+  fi
+  shift
+fi
+
+bin="$(dirname "${BASH_SOURCE-$0}")"
+bin="$(cd "${bin}">/dev/null; pwd)"
+
+. "${bin}/common.sh"
+
+check_java_version
+
+function check_process_status() {
+  local pid=$(found_lance_rest_server_pid)
+
+  if [[ -z "${pid}" ]]; then
+    echo "GravitinoLanceRESTServer is not running"
+  else
+    printArt
+    echo "GravitinoLanceRESTServer is running[PID:$pid]"
+  fi
+}
+
+function found_lance_rest_server_pid() {
+  process_name='GravitinoLanceRESTServer';
+  RUNNING_PIDS=$(ps x | grep ${process_name} | grep -v grep | awk '{print $1}');
+
+  if [[ -z "${RUNNING_PIDS}" ]]; then
+    return
+  fi
+
+  if ! kill -0 ${RUNNING_PIDS} > /dev/null 2>&1; then
+    echo "GravitinoLanceRESTServer running but process is dead"
+  fi
+
+  echo "${RUNNING_PIDS}"
+}
+
+function wait_for_lance_rest_server_to_die() {
+  timeout=10
+  timeoutTime=$(date "+%s")
+  let "timeoutTime+=$timeout"
+  currentTime=$(date "+%s")
+  forceKill=1
+
+  while [[ $currentTime -lt $timeoutTime ]]; do
+    local pid=$(found_lance_rest_server_pid)
+    if [[ -z "${pid}" ]]; then
+      forceKill=0
+      break
+    fi
+
+    kill ${pid} > /dev/null 2> /dev/null
+    if kill -0 ${pid} > /dev/null 2>&1; then
+      sleep 3
+    else
+      forceKill=0
+      break
+    fi
+    currentTime=$(date "+%s")
+  done
+
+  if [[ $forceKill -ne 0 ]]; then
+    kill -9 ${pid} > /dev/null 2> /dev/null
+  fi
+}
+
+function start() {
+  local pid=$(found_lance_rest_server_pid)
+
+  if [[ ! -z "${pid}" ]]; then
+    if kill -0 ${pid} >/dev/null 2>&1; then
+      echo "GravitinoLanceRESTServer is already running"
+      return 0;
+    fi
+  fi
+
+  if [[ ! -d "${GRAVITINO_LOG_DIR}" ]]; then
+    echo "Log dir doesn't exist, create ${GRAVITINO_LOG_DIR}"
+    mkdir -p "${GRAVITINO_LOG_DIR}"
+  fi
+
+  nohup ${JAVA_RUNNER} ${JAVA_OPTS} ${GRAVITINO_DEBUG_OPTS} -cp ${GRAVITINO_CLASSPATH} ${GRAVITINO_SERVER_NAME} >> "${GRAVITINO_OUTFILE}" 2>&1 &
+
+  pid=$!
+  if [[ -z "${pid}" ]]; then
+    echo "GravitinoLanceRESTServer start error!"
+    return 1;
+  fi
+
+  sleep 2
+  check_process_status
+}
+
+function run() {
+  ${JAVA_RUNNER} ${JAVA_OPTS} ${GRAVITINO_DEBUG_OPTS} -cp ${GRAVITINO_CLASSPATH} ${GRAVITINO_SERVER_NAME}
+}
+
+function stop() {
+  local pid
+
+  pid=$(found_lance_rest_server_pid)
+
+  if [[ -z "${pid}" ]]; then
+    echo "GravitinoLanceRESTServer is not running"
+  else
+    wait_for_lance_rest_server_to_die
+    echo "GravitinoLanceRESTServer stop"
+  fi
+}
+
+HOSTNAME=$(hostname)
+GRAVITINO_OUTFILE="${GRAVITINO_LOG_DIR}/gravitino-lance-rest-server.out"
+GRAVITINO_SERVER_NAME=org.apache.gravitino.lance.server.GravitinoLanceRESTServer
+GRAVITINO_SIMPLE_SERVER_NAME=gravitino-lance-rest-server
+
+JAVA_OPTS+=" -Dfile.encoding=UTF-8"
+JAVA_OPTS+=" -Dlog4j2.configurationFile=file://${GRAVITINO_CONF_DIR}/log4j2.properties"
+JAVA_OPTS+=" -Dgravitino.log.path=${GRAVITINO_LOG_DIR} ${GRAVITINO_MEM}"
+JAVA_OPTS+=" -Dgravitino.server.name=${GRAVITINO_SIMPLE_SERVER_NAME}"
+if [ "$JVM_VERSION" -eq 17 ]; then
+  JAVA_OPTS+=" -XX:+IgnoreUnrecognizedVMOptions"
+  JAVA_OPTS+=" --add-opens java.base/java.io=ALL-UNNAMED"
+  JAVA_OPTS+=" --add-opens java.base/java.lang.invoke=ALL-UNNAMED"
+  JAVA_OPTS+=" --add-opens java.base/java.lang.reflect=ALL-UNNAMED"
+  JAVA_OPTS+=" --add-opens java.base/java.lang=ALL-UNNAMED"
+  JAVA_OPTS+=" --add-opens java.base/java.math=ALL-UNNAMED"
+  JAVA_OPTS+=" --add-opens java.base/java.net=ALL-UNNAMED"
+  JAVA_OPTS+=" --add-opens java.base/java.nio=ALL-UNNAMED"
+  JAVA_OPTS+=" --add-opens java.base/java.text=ALL-UNNAMED"
+  JAVA_OPTS+=" --add-opens java.base/java.time=ALL-UNNAMED"
+  JAVA_OPTS+=" --add-opens java.base/java.util.concurrent.atomic=ALL-UNNAMED"
+  JAVA_OPTS+=" --add-opens java.base/java.util.concurrent=ALL-UNNAMED"
+  JAVA_OPTS+=" --add-opens java.base/java.util.regex=ALL-UNNAMED"
+  JAVA_OPTS+=" --add-opens java.base/java.util=ALL-UNNAMED"
+  JAVA_OPTS+=" --add-opens java.base/jdk.internal.ref=ALL-UNNAMED"
+  JAVA_OPTS+=" --add-opens java.base/jdk.internal.reflect=ALL-UNNAMED"
+  JAVA_OPTS+=" --add-opens java.sql/java.sql=ALL-UNNAMED"
+  JAVA_OPTS+=" --add-opens java.base/sun.util.calendar=ALL-UNNAMED"
+  JAVA_OPTS+=" --add-opens java.base/sun.nio.ch=ALL-UNNAMED"
+  JAVA_OPTS+=" --add-opens java.base/sun.nio.cs=ALL-UNNAMED"
+  JAVA_OPTS+=" --add-opens java.base/sun.security.action=ALL-UNNAMED"
+  JAVA_OPTS+=" --add-opens java.base/sun.util.calendar=ALL-UNNAMED"
+  JAVA_OPTS+=" --add-opens java.security.jgss/sun.security.krb5=ALL-UNNAMED"
+fi
+
+#JAVA_OPTS+=" -Djava.security.krb5.conf=/etc/krb5.conf"
+
+if [ -d "${GRAVITINO_HOME}/lance-rest-server/libs" ]; then
+  addJarInDir "${GRAVITINO_HOME}/lance-rest-server/libs"
+  addDirToClasspath "${GRAVITINO_HOME}/lance-rest-server/conf"
+else
+  addJarInDir "${GRAVITINO_HOME}/libs"
+fi
+
+case "${1}" in
+  start)
+    start
+    ;;
+  run)
+    run
+    ;;
+  stop)
+    stop
+    ;;
+  restart)
+    stop
+    start
+    ;;
+  status)
+    check_process_status
+    ;;
+  *)
+    echo ${USAGE}
+esac

build.gradle.kts

@@ -683,6 +683,7 @@ tasks {
       "copyCliLib",
       ":authorizations:copyLibAndConfig",
       ":iceberg:iceberg-rest-server:copyLibAndConfigs",
+      ":lance:lance-rest-server:copyLibAndConfigs",
       ":web:web:build"
     )
 
@@ -772,6 +773,51 @@ tasks {
     }
   }
 
+  val compileLanceRESTServer by registering {
+    dependsOn("lance:lance-rest-server:copyLibAndConfigsToStandalonePackage")
+    group = "gravitino distribution"
+    outputs.dir(projectDir.dir("distribution/${rootProject.name}-lance-rest-server"))
+    doLast {
+      copy {
+        from(projectDir.dir("conf")) {
+          include(
+            "${rootProject.name}-lance-rest-server.conf.template",
+            "${rootProject.name}-env.sh.template",
+            "log4j2.properties.template"
+          )
+          into("${rootProject.name}-lance-rest-server/conf")
+        }
+        from(projectDir.dir("bin")) {
+          include("common.sh.template", "${rootProject.name}-lance-rest-server.sh.template")
+          into("${rootProject.name}-lance-rest-server/bin")
+        }
+        into(outputDir)
+        rename { fileName ->
+          fileName.replace(".template", "")
+        }
+        eachFile {
+          if (name == "gravitino-env.sh") {
+            filter { line ->
+              line.replace("GRAVITINO_VERSION_PLACEHOLDER", "$version")
+            }
+          }
+        }
+        fileMode = 0b111101101
+      }
+
+      copy {
+        from(projectDir.dir("licenses")) { into("${rootProject.name}-lance-rest-server/licenses") }
+        from(projectDir.file("LICENSE.rest")) { into("${rootProject.name}-lance-rest-server") }
+        from(projectDir.file("NOTICE.rest")) { into("${rootProject.name}-lance-rest-server") }
+        from(projectDir.file("README.md")) { into("${rootProject.name}-lance-rest-server") }
+        into(outputDir)
+        rename { fileName ->
+          fileName.replace(".rest", "")
+        }
+      }
+    }
+  }
+
   val compileTrinoConnector by registering {
     dependsOn("trino-connector:trino-connector:copyLibs")
     group = "gravitino distribution"
@@ -791,7 +837,7 @@ tasks {
   }
 
   val assembleDistribution by registering(Tar::class) {
-    dependsOn("assembleTrinoConnector", "assembleIcebergRESTServer")
+    dependsOn("assembleTrinoConnector", "assembleIcebergRESTServer", "assembleLanceRESTServer")
     group = "gravitino distribution"
     finalizedBy("checksumDistribution")
     into("${rootProject.name}-$version-bin")
@@ -823,6 +869,17 @@ tasks {
     destinationDirectory.set(projectDir.dir("distribution"))
   }
 
+  val assembleLanceRESTServer by registering(Tar::class) {
+    dependsOn("compileLanceRESTServer")
+    group = "gravitino distribution"
+    finalizedBy("checksumLanceRESTServerDistribution")
+    into("${rootProject.name}-lance-rest-server-$version-bin")
+    from(compileLanceRESTServer.map { it.outputs.files.single() })
+    compression = Compression.GZIP
+    archiveFileName.set("${rootProject.name}-lance-rest-server-$version-bin.tar.gz")
+    destinationDirectory.set(projectDir.dir("distribution"))
+  }
+
   register("checksumIcebergRESTServerDistribution") {
     group = "gravitino distribution"
     dependsOn(assembleIcebergRESTServer)
@@ -839,9 +896,30 @@ tasks {
     }
   }
 
+  register("checksumLanceRESTServerDistribution") {
+    group = "gravitino distribution"
+    dependsOn(assembleLanceRESTServer)
+    val archiveFile = assembleLanceRESTServer.flatMap { it.archiveFile }
+    val checksumFile = archiveFile.map { archive ->
+      archive.asFile.let { it.resolveSibling("${it.name}.sha256") }
+    }
+    inputs.file(archiveFile)
+    outputs.file(checksumFile)
+    doLast {
+      checksumFile.get().writeText(
+        serviceOf<ChecksumService>().sha256(archiveFile.get().asFile).toString()
+      )
+    }
+  }
+
   register("checksumDistribution") {
     group = "gravitino distribution"
-    dependsOn(assembleDistribution, "checksumTrinoConnector", "checksumIcebergRESTServerDistribution")
+    dependsOn(
+      assembleDistribution,
+      "checksumTrinoConnector",
+      "checksumIcebergRESTServerDistribution",
+      "checksumLanceRESTServerDistribution"
+    )
     val archiveFile = assembleDistribution.flatMap { it.archiveFile }
     val checksumFile = archiveFile.map { archive ->
       archive.asFile.let { it.resolveSibling("${it.name}.sha256") }
@@ -885,6 +963,7 @@ tasks {
         !it.name.startsWith("filesystem") &&
         !it.name.startsWith("flink") &&
         !it.name.startsWith("iceberg") &&
+        !it.name.startsWith("lance") &&
         !it.name.startsWith("spark") &&
         it.name != "hadoop-common" &&
         it.name != "hive-metastore-common" &&
@@ -916,6 +995,7 @@ tasks {
         !it.name.startsWith("filesystem") &&
         !it.name.startsWith("flink") &&
         !it.name.startsWith("iceberg") &&
+        !it.name.startsWith("lance") &&
         !it.name.startsWith("integration-test") &&
         !it.name.startsWith("spark") &&
         !it.name.startsWith("trino-connector") &&

conf/gravitino-lance-rest-server.conf.template

@@ -0,0 +1,45 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# THE CONFIGURATION FOR Lance REST SERVER
+gravitino.lance-rest.shutdown.timeout = 3000
+
+# THE CONFIGURATION FOR Lance REST WEB SERVER
+# The host name of the built-in web server
+gravitino.lance-rest.host = 0.0.0.0
+# The http port number of the built-in web server
+gravitino.lance-rest.httpPort = 9101
+# The min thread size of the built-in web server
+gravitino.lance-rest.minThreads = 24
+# The max thread size of the built-in web server
+gravitino.lance-rest.maxThreads = 200
+# The stop timeout of the built-in web server
+gravitino.lance-rest.stopTimeout = 30000
+# The timeout of idle connections
+gravitino.lance-rest.idleTimeout = 30000
+# The executor thread pool work queue size of the built-in web server
+gravitino.lance-rest.threadPoolWorkQueueSize = 100
+# The request header size of the built-in web server
+gravitino.lance-rest.requestHeaderSize = 131072
+# The response header size of the built-in web server
+gravitino.lance-rest.responseHeaderSize = 131072
+
+# THE CONFIGURATION FOR Lance CATALOG
+# The logical Lance catalog served by this REST endpoint
+gravitino.lance-rest.catalog-name = default