enhance: not build lindera features by default and support make milvus with tantivy features (milvus-io#40813)

relate: milvus-io#39659

Signed-off-by: aoiasd <[email protected]>

Author: aoiasd

Makefile

@@ -40,6 +40,11 @@ ifdef USE_DYNAMIC_SIMD
 	use_dynamic_simd = ${USE_DYNAMIC_SIMD}
 endif
 
+tantivy_features = ""
+ifdef TANTIVY_FEATURES
+	tantivy_features = ${TANTIVY_FEATURES}
+endif
+
 use_opendal = OFF
 ifdef USE_OPENDAL
 	use_opendal = ${USE_OPENDAL}
@@ -260,19 +265,19 @@ generated-proto: download-milvus-proto build-3rdparty get-proto-deps
 
 build-cpp: generated-proto
 	@echo "Building Milvus cpp library ..."
-	@(env bash $(PWD)/scripts/core_build.sh -t ${mode} -a ${use_asan} -n ${use_disk_index} -y ${use_dynamic_simd} ${AZURE_OPTION} -x ${index_engine} -o ${use_opendal})
+	@(env bash $(PWD)/scripts/core_build.sh -t ${mode} -a ${use_asan} -n ${use_disk_index} -y ${use_dynamic_simd} ${AZURE_OPTION} -x ${index_engine} -o ${use_opendal} -f $(tantivy_features))
 
 build-cpp-gpu: generated-proto
 	@echo "Building Milvus cpp gpu library ... "
-	@(env bash $(PWD)/scripts/core_build.sh -t ${mode} -g -n ${use_disk_index} -y ${use_dynamic_simd} ${AZURE_OPTION} -x ${index_engine} -o ${use_opendal})
+	@(env bash $(PWD)/scripts/core_build.sh -t ${mode} -g -n ${use_disk_index} -y ${use_dynamic_simd} ${AZURE_OPTION} -x ${index_engine} -o ${use_opendal} -f $(tantivy_features))
 
 build-cpp-with-unittest: generated-proto
 	@echo "Building Milvus cpp library with unittest ... "
-	@(env bash $(PWD)/scripts/core_build.sh -t ${mode} -a ${use_asan} -u -n ${use_disk_index} -y ${use_dynamic_simd} ${AZURE_OPTION} -x ${index_engine} -o ${use_opendal})
+	@(env bash $(PWD)/scripts/core_build.sh -t ${mode} -a ${use_asan} -u -n ${use_disk_index} -y ${use_dynamic_simd} ${AZURE_OPTION} -x ${index_engine} -o ${use_opendal} -f $(tantivy_features))
 
 build-cpp-with-coverage: generated-proto
 	@echo "Building Milvus cpp library with coverage and unittest ..."
-	@(env bash $(PWD)/scripts/core_build.sh -t ${mode} -a ${use_asan} -u -c -n ${use_disk_index} -y ${use_dynamic_simd} ${AZURE_OPTION} -x ${index_engine} -o ${use_opendal})
+	@(env bash $(PWD)/scripts/core_build.sh -t ${mode} -a ${use_asan} -u -c -n ${use_disk_index} -y ${use_dynamic_simd} ${AZURE_OPTION} -x ${index_engine} -o ${use_opendal} -f $(tantivy_features))
 
 check-proto-product: generated-proto
 	 @(env bash $(PWD)/scripts/check_proto_product.sh)

internal/core/thirdparty/tantivy/CMakeLists.txt

@@ -1,3 +1,6 @@
+set(TANTIVY_FEATURES_LIST "" CACHE STRING "List of Cargo features to enable")
+string(REPLACE ";" "," TANTIVY_FEATURES "${TANTIVY_FEATURES_LIST}")
+
 if (CMAKE_BUILD_TYPE STREQUAL "Debug")
     set(CARGO_CMD cargo build)
     set(TARGET_DIR "debug")
@@ -6,6 +9,11 @@ else ()
     set(TARGET_DIR "release")
 endif ()
 
+if (TANTIVY_FEATURES)
+    set(CARGO_CMD ${CARGO_CMD} --features ${TANTIVY_FEATURES})
+endif ()
+message("Cargo command: ${CARGO_CMD}")
+
 set(TANTIVY_LIB_DIR "${CMAKE_INSTALL_PREFIX}/lib")
 set(TANTIVY_INCLUDE_DIR "${CMAKE_INSTALL_PREFIX}/include")
 set(TANTIVY_NAME "libtantivy_binding${CMAKE_STATIC_LIBRARY_SUFFIX}")

internal/core/thirdparty/tantivy/tantivy-binding/Cargo.lock

@@ -6,7 +6,7 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [features]
-default = ["lindera-ipadic", "lindera-ko-dic", "lindera-cc-cedict"]
+default = []
 lindera-ipadic = ["lindera/ipadic"]
 lindera-ipadic-neologd = ["lindera/ipadic-neologd"]
 lindera-unidic = ["lindera/unidic"]
@@ -15,7 +15,7 @@ lindera-cc-cedict = ["lindera/cc-cedict"]
 
 [dependencies]
 tantivy = { git = "https://github.com/zilliztech/tantivy.git" }
-lindera = "0.38.1"
+lindera = "0.40.1"
 futures = "0.3.21"
 libc = "0.2"
 scopeguard = "1.2"

internal/core/thirdparty/tantivy/tantivy-binding/Cargo.toml

@@ -971,7 +971,6 @@ func (s *DelegatorDataSuite) TestBuildBM25IDF() {
 			newGrow = append(newGrow, SegmentEntry{NodeID: 1, SegmentID: seg, TargetVersion: targetVersion})
 		}
 
-		log.Info("Test-", zap.Any("shanshot", snapshot), zap.Any("seg", newSeal))
 		snapshot.dist[0].Segments = newSeal
 		snapshot.growing = newGrow
 		return snapshot

internal/querynodev2/delegator/delegator_data_test.go

@@ -100,11 +100,12 @@ BUILD_DISK_ANN="OFF"
 USE_ASAN="OFF"
 USE_DYNAMIC_SIMD="ON"
 USE_OPENDAL="OFF"
+TANTIVY_FEATURES=""
 INDEX_ENGINE="KNOWHERE"
 ENABLE_AZURE_FS="OFF"
 : "${ENABLE_GCP_NATIVE:="OFF"}"
 
-while getopts "p:d:t:s:f:n:i:y:a:x:o:ulrcghzmebZ" arg; do
+while getopts "p:t:u:l:c:g:s:b:n:a:y:Z:x:o:f:" arg; do
   case $arg in
   p)
     INSTALL_PREFIX=$OPTARG
@@ -153,6 +154,9 @@ while getopts "p:d:t:s:f:n:i:y:a:x:o:ulrcghzmebZ" arg; do
   o)
     USE_OPENDAL=$OPTARG
     ;;
+  f)
+    TANTIVY_FEATURES=$OPTARG
+    ;;
   h) # help
     echo "
 
@@ -170,10 +174,11 @@ parameter:
 -a: build milvus with AddressSanitizer(default: false)
 -Z: build milvus without azure-sdk-for-cpp, so cannot use azure blob
 -o: build milvus with opendal(default: false)
+-f: build milvus with tantivy features(default: '')
 -h: help
 
 usage:
-./core_build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} -s \${CUDA_ARCH} [-u] [-l] [-r] [-c] [-z] [-g] [-m] [-e] [-h] [-b] [-o]
+./core_build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} -s \${CUDA_ARCH} -f \${TANTIVY_FEATURES} [-u] [-l] [-r] [-c] [-z] [-g] [-m] [-e] [-h] [-b] [-o]
                 "
     exit 0
     ;;
@@ -258,6 +263,7 @@ ${CMAKE_EXTRA_ARGS} \
 -DCPU_ARCH=${CPU_ARCH} \
 -DUSE_OPENDAL=${USE_OPENDAL} \
 -DINDEX_ENGINE=${INDEX_ENGINE} \
+-DTANTIVY_FEATURES_LIST=${TANTIVY_FEATURES} \
 -DENABLE_GCP_NATIVE=${ENABLE_GCP_NATIVE} \
 -DENABLE_AZURE_FS=${ENABLE_AZURE_FS} "
 if [ -z "$BUILD_WITHOUT_AZURE" ]; then