SPFresh implementation based on SPTAG (#405)

.gitmodules

@@ -2,3 +2,12 @@
 	path = ThirdParty/zstd
 	url = https://github.com/facebook/zstd
 	branch = release
+[submodule "ThirdParty/spdk"]
+	path = ThirdParty/spdk
+	url = https://github.com/spdk/spdk
+[submodule "ThirdParty/isal-l_crypto"]
+	path = ThirdParty/isal-l_crypto
+	url = https://github.com/intel/isa-l_crypto
+[submodule "ThirdParty/RocksDB"]
+	path = ThirdParty/RocksDB
+	url = [email protected]:facebook/rocksdb.git

AnnService/CMakeLists.txt

@@ -3,9 +3,16 @@
 
 set(AnnService ${PROJECT_SOURCE_DIR}/AnnService)
 set(Zstd ${PROJECT_SOURCE_DIR}/ThirdParty/zstd)
+set(Spdk ${PROJECT_SOURCE_DIR}/ThirdParty/spdk/build)
+set(Dpdk ${PROJECT_SOURCE_DIR}/ThirdParty/spdk/dpdk/build)
+set(IsalLCrypto ${PROJECT_SOURCE_DIR}/ThirdParty/isal-l_crypto/.libs/libisal_crypto.a)
+set(SpdkLibPrefix ${Spdk}/lib/libspdk_)
+set(DpdkLibPrefix ${Dpdk}/lib/librte_)
+set(SPDK_LIBRARIES -Wl,--whole-archive ${SpdkLibPrefix}bdev_nvme.a ${SpdkLibPrefix}bdev.a ${SpdkLibPrefix}nvme.a ${SpdkLibPrefix}vfio_user.a ${SpdkLibPrefix}sock.a ${SpdkLibPrefix}dma.a ${SpdkLibPrefix}notify.a ${SpdkLibPrefix}accel.a ${SpdkLibPrefix}event_bdev.a ${SpdkLibPrefix}event_accel.a ${SpdkLibPrefix}vmd.a ${SpdkLibPrefix}event_vmd.a ${SpdkLibPrefix}event_sock.a ${SpdkLibPrefix}event_iobuf.a ${SpdkLibPrefix}event.a ${SpdkLibPrefix}env_dpdk.a ${SpdkLibPrefix}log.a ${SpdkLibPrefix}thread.a ${SpdkLibPrefix}rpc.a ${SpdkLibPrefix}init.a ${SpdkLibPrefix}jsonrpc.a ${SpdkLibPrefix}json.a ${SpdkLibPrefix}trace.a ${SpdkLibPrefix}util.a ${DpdkLibPrefix}mempool.a ${DpdkLibPrefix}mempool_ring.a ${DpdkLibPrefix}eal.a ${DpdkLibPrefix}ring.a ${DpdkLibPrefix}telemetry.a ${DpdkLibPrefix}bus_pci.a ${DpdkLibPrefix}kvargs.a ${DpdkLibPrefix}pci.a -Wl,--no-whole-archive dl rt isal ${IsalLCrypto} uuid)
 
 include_directories(${AnnService})
 include_directories(${Zstd}/lib)
+include_directories(${Spdk}/include)
 
 file(GLOB_RECURSE HDR_FILES ${AnnService}/inc/Core/*.h  ${AnnService}/inc/Helper/*.h)
 file(GLOB_RECURSE SRC_FILES ${AnnService}/src/Core/*.cpp ${AnnService}/src/Helper/*.cpp)
@@ -37,10 +44,28 @@ if(${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU")
     target_compile_options(DistanceUtils PRIVATE -mavx2 -mavx -msse -msse2 -mavx512f -mavx512bw -mavx512dq -fPIC)
 endif()
 
+find_package(RocksDB CONFIG)
+if((DEFINED RocksDB_DIR) AND RocksDB_DIR)
+    list(APPEND RocksDB_LIBRARIES RocksDB::rocksdb)
+    find_package(uring)
+    set(uring_LIBRARIES "")
+    message (STATUS "Found RocksDB ${RocksDB_VERSION}")
+    message (STATUS "RocksDB: ${RocksDB_DIR}")
+else()
+    set(RocksDB_LIBRARIES "")
+    set(uring_LIBRARIES "")
+    message (FATAL_ERROR "Could not find RocksDB!")
+endif()
+
+find_package(TBB REQUIRED)
+if (TBB_FOUND)
+    message (STATUS "Found TBB")
+endif()
+
 add_library (SPTAGLib SHARED ${SRC_FILES} ${HDR_FILES})
-target_link_libraries (SPTAGLib DistanceUtils libzstd_shared ${NUMA_LIBRARY})
+target_link_libraries (SPTAGLib DistanceUtils ${RocksDB_LIBRARIES} ${uring_LIBRARIES} libzstd_shared ${NUMA_LIBRARY} tbb ${SPDK_LIBRARIES})
 add_library (SPTAGLibStatic STATIC ${SRC_FILES} ${HDR_FILES})
-target_link_libraries (SPTAGLibStatic DistanceUtils libzstd_static ${NUMA_LIBRARY_STATIC})
+target_link_libraries (SPTAGLibStatic DistanceUtils ${RocksDB_LIBRARIES} ${uring_LIBRARIES} libzstd_static ${NUMA_LIBRARY_STATIC} tbb ${SPDK_LIBRARIES})
 if(${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU")
     target_compile_options(SPTAGLibStatic PRIVATE -fPIC)
 endif()
@@ -88,9 +113,15 @@ endif()
 file(GLOB_RECURSE SSD_SERVING_HDR_FILES ${AnnService}/inc/SSDServing/*.h)
 file(GLOB_RECURSE SSD_SERVING_FILES ${AnnService}/src/SSDServing/*.cpp)
 
+file(GLOB_RECURSE SPFRESH_HDR_FILES ${AnnService}/inc/SPFresh/*.h)
+file(GLOB_RECURSE SPFRESH_FILES ${AnnService}/src/SPFresh/*.cpp)
+
 add_executable(ssdserving ${SSD_SERVING_HDR_FILES} ${SSD_SERVING_FILES})
-target_link_libraries(ssdserving SPTAGLibStatic ${Boost_LIBRARIES})
+add_executable(spfresh ${SPFRESH_HDR_FILES} ${SPFRESH_FILES})
+target_link_libraries(ssdserving SPTAGLibStatic ${Boost_LIBRARIES} ${RocksDB_LIBRARIES})
+target_link_libraries(spfresh SPTAGLibStatic ${Boost_LIBRARIES} ${RocksDB_LIBRARIES})
 target_compile_definitions(ssdserving PRIVATE _exe)
+target_compile_definitions(spfresh PRIVATE _exe)
 
 # for Test
 add_library(ssdservingLib ${SSD_SERVING_HDR_FILES} ${SSD_SERVING_FILES})

AnnService/CoreLibrary.vcxproj

@@ -149,6 +149,7 @@
     <ClInclude Include="inc\Core\Common\IQuantizer.h" />
     <ClInclude Include="inc\Core\Common\SIMDUtils.h" />
     <ClInclude Include="inc\Core\Common\TruthSet.h" />
+    <ClInclude Include="inc\Core\Common\VersionLabel.h" />
     <ClInclude Include="inc\Core\Common\WorkSpace.h" />
     <ClInclude Include="inc\Core\Common\CommonUtils.h" />
     <ClInclude Include="inc\Core\Common\Dataset.h" />
@@ -167,7 +168,10 @@
     <ClInclude Include="inc\Core\SearchQuery.h" />
     <ClInclude Include="inc\Core\SearchResult.h" />
     <ClInclude Include="inc\Core\SPANN\Compressor.h" />
-    <ClInclude Include="inc\Core\SPANN\ExtraFullGraphSearcher.h" />
+    <ClInclude Include="inc\Core\SPANN\ExtraDynamicSearcher.h" />
+    <ClInclude Include="inc\Core\SPANN\ExtraSPDKController.h" />
+    <ClInclude Include="inc\Core\SPANN\ExtraStaticSearcher.h" />
+    <ClInclude Include="inc\Core\SPANN\ExtraRocksDBController.h" />
     <ClInclude Include="inc\Core\SPANN\IExtraSearcher.h" />
     <ClInclude Include="inc\Core\SPANN\Index.h" />
     <ClInclude Include="inc\Core\SPANN\Options.h" />
@@ -182,6 +186,7 @@
     <ClInclude Include="inc\Helper\ConcurrentSet.h" />
     <ClInclude Include="inc\Helper\DiskIO.h" />
     <ClInclude Include="inc\Helper\DynamicNeighbors.h" />
+    <ClInclude Include="inc\Helper\KeyValueIO.h" />
     <ClInclude Include="inc\Helper\LockFree.h" />
     <ClInclude Include="inc\Helper\Logging.h" />
     <ClInclude Include="inc\Helper\SimpleIniReader.h" />
@@ -229,11 +234,15 @@
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
     <Import Project="..\packages\Zstandard.dyn.x64.1.4.0\build\native\Zstandard.dyn.x64.targets" Condition="Exists('..\packages\Zstandard.dyn.x64.1.4.0\build\native\Zstandard.dyn.x64.targets')" />
+    <Import Project="..\packages\tbb.redist.4.2.3.1\build\native\tbb.redist.targets" Condition="Exists('..\packages\tbb.redist.4.2.3.1\build\native\tbb.redist.targets')" />
+    <Import Project="..\packages\tbb.4.2.3.1\build\native\tbb.targets" Condition="Exists('..\packages\tbb.4.2.3.1\build\native\tbb.targets')" />
   </ImportGroup>
   <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
     <PropertyGroup>
       <ErrorText>This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them.  For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
     </PropertyGroup>
     <Error Condition="!Exists('..\packages\Zstandard.dyn.x64.1.4.0\build\native\Zstandard.dyn.x64.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Zstandard.dyn.x64.1.4.0\build\native\Zstandard.dyn.x64.targets'))" />
+    <Error Condition="!Exists('..\packages\tbb.redist.4.2.3.1\build\native\tbb.redist.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\tbb.redist.4.2.3.1\build\native\tbb.redist.targets'))" />
+    <Error Condition="!Exists('..\packages\tbb.4.2.3.1\build\native\tbb.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\tbb.4.2.3.1\build\native\tbb.targets'))" />
   </Target>
 </Project>

AnnService/CoreLibrary.vcxproj.filters

@@ -196,7 +196,7 @@
     <ClInclude Include="inc\Core\SPANN\Options.h">
       <Filter>Header Files\Core\SPANN</Filter>
     </ClInclude>
-    <ClInclude Include="inc\Core\SPANN\ExtraFullGraphSearcher.h">
+    <ClInclude Include="inc\Core\SPANN\ExtraStaticSearcher.h">
       <Filter>Header Files\Core\SPANN</Filter>
     </ClInclude>
     <ClInclude Include="inc\Helper\AsyncFileReader.h">
@@ -220,6 +220,21 @@
     <ClInclude Include="inc\Core\Common\SIMDUtils.h">
       <Filter>Header Files\Core\Common</Filter>
     </ClInclude>
+    <ClInclude Include="inc\Core\SPANN\ExtraRocksDBController.h">
+      <Filter>Header Files\Core\SPANN</Filter>
+    </ClInclude>
+    <ClInclude Include="inc\Core\SPANN\ExtraDynamicSearcher.h">
+      <Filter>Header Files\Core\SPANN</Filter>
+    </ClInclude>
+    <ClInclude Include="inc\Helper\KeyValueIO.h">
+      <Filter>Header Files\Helper</Filter>
+    </ClInclude>
+    <ClInclude Include="inc\Core\Common\VersionLabel.h">
+      <Filter>Header Files\Core\Common</Filter>
+    </ClInclude>
+    <ClInclude Include="inc\Core\SPANN\ExtraSPDKController.h">
+      <Filter>Header Files\Core\SPANN</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="src\Core\VectorIndex.cpp">

AnnService/inc/Core/BKT/Index.h

@@ -178,6 +178,8 @@ namespace SPTAG
             ErrorCode RefineSearchIndex(QueryResult &p_query, bool p_searchDeleted = false) const;
             ErrorCode SearchTree(QueryResult &p_query) const;
             ErrorCode AddIndex(const void* p_data, SizeType p_vectorNum, DimensionType p_dimension, std::shared_ptr<MetadataSet> p_metadataSet, bool p_withMetaIndex = false, bool p_normalized = false);
+            ErrorCode AddIndexIdx(SizeType begin, SizeType end);
+            ErrorCode AddIndexId(const void* p_data, SizeType p_vectorNum, DimensionType p_dimension, int& beginHead, int& endHead);
             ErrorCode DeleteIndex(const void* p_vectors, SizeType p_vectorNum);
             ErrorCode DeleteIndex(const SizeType& p_id);
 
@@ -209,7 +211,6 @@ namespace SPTAG
 
         private:
             void SearchIndex(COMMON::QueryResultSet<T> &p_query, COMMON::WorkSpace &p_space, bool p_searchDeleted, bool p_searchDuplicated, std::function<bool(const ByteArray&)> filterFunc = nullptr) const;
-
             template <bool(*notDeleted)(const COMMON::Labelset&, SizeType), bool(*isDup)(COMMON::QueryResultSet<T>&, SizeType, float), bool(*checkFilter)(const std::shared_ptr<MetadataSet>&, SizeType, std::function<bool(const ByteArray&)>)>
             void Search(COMMON::QueryResultSet<T>& p_query, COMMON::WorkSpace& p_space, std::function<bool(const ByteArray&)> filterFunc) const;
         };

AnnService/inc/Core/Common.h

@@ -67,6 +67,7 @@ inline T max(T a, T b) {
 #define InterlockedExchange8(a,b) __sync_lock_test_and_set(a, b)
 #define Sleep(a) usleep(a * 1000)
 #define strtok_s(a, b, c) strtok_r(a, b, c)
+#define ALIGN_ROUND(size) ((size) + 31) / 32 * 32
 
 #else
 
@@ -98,6 +99,17 @@ inline bool fileexists(const TCHAR* path) {
 #define min(a,b) (((a) < (b)) ? (a) : (b))
 #endif
 
+FORCEINLINE
+char
+InterlockedCompareExchange(
+    _Inout_ _Interlocked_operand_ char volatile* Destination,
+    _In_ char Exchange,
+    _In_ char Comperand
+)
+{
+    return (char)_InterlockedCompareExchange8(Destination, Exchange, Comperand);
+}
+
 #endif
 
 namespace SPTAG
@@ -114,6 +126,8 @@ namespace SPTAG
 #define PAGE_FREE(ptr) ::operator delete(ptr, (std::align_val_t)512)
 #endif
 
+#define ALIGN_ROUND(size) ((size) + 31) / 32 * 32
+
 typedef std::int32_t SizeType;
 typedef std::int32_t DimensionType;