add LICENSE (#1)

* add LICENSE * new release * add url
microsoft · Dec 29, 2018 · 2e7c61c · 2e7c61c
1 parent 5bd0182
commit 2e7c61c
Show file tree

Hide file tree

Showing 65 changed files with 3,072 additions and 1,258 deletions.
diff --git a/.gitignore b/.gitignore
@@ -71,3 +71,4 @@ dkms.conf
 /obj/x64_Debug
 /x64/Debug
 /packages
+/Search/Search.vcxproj.user
diff --git a/AnnService/CMakeLists.txt b/AnnService/CMakeLists.txt
@@ -1,25 +1,3 @@
-find_package(OpenMP)
-if (OpenMP_FOUND)
-    set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}")
-    set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
-    set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OpenMP_EXE_LINKER_FLAGS}")
-    message (STATUS "Found openmp.")
-else()
-    message (FATAL_ERROR "Could no find openmp!")
-endif()
-
-find_package(Boost 1.67 COMPONENTS system thread serialization wserialization regex)
-if (Boost_FOUND)
-    include_directories (${Boost_INCLUDE_DIR})
-    link_directories (${Boost_LIBRARY_DIR} "/usr/lib")
-    message (STATUS "Found Boost.")
-    message (STATUS "Include Path: ${Boost_INCLUDE_DIRS}")
-    message (STATUS "Library Path: ${Boost_LIBRARY_DIRS}")
-    message (STATUS "Library: ${Boost_LIBRARIES}")
-else()
-    message (FATAL_ERROR "Could not find Boost 1.67!")
-endif()
-
 file(GLOB HDR_FILES ${PROJECT_SOURCE_DIR}/AnnService/inc/Core/*.h ${PROJECT_SOURCE_DIR}/AnnService/inc/Core/Common/*.h ${PROJECT_SOURCE_DIR}/AnnService/inc/Core/BKT/*.h ${PROJECT_SOURCE_DIR}/AnnService/inc/Core/KDT/*.h ${PROJECT_SOURCE_DIR}/AnnService/inc/Helper/*.h)
 file(GLOB SRC_FILES ${PROJECT_SOURCE_DIR}/AnnService/src/Core/*.cpp ${PROJECT_SOURCE_DIR}/AnnService/src/Core/Common/*.cpp ${PROJECT_SOURCE_DIR}/AnnService/src/Core/BKT/*.cpp ${PROJECT_SOURCE_DIR}/AnnService/src/Core/KDT/*.cpp ${PROJECT_SOURCE_DIR}/AnnService/src/Helper/*.cpp)
 
@@ -32,24 +10,24 @@ set_target_properties(SPTAGLibStatic PROPERTIES OUTPUT_NAME SPTAGLib)
 file(GLOB SERVER_HDR_FILES ${HDR_FILES} ${PROJECT_SOURCE_DIR}/AnnService/inc/Server/*.h ${PROJECT_SOURCE_DIR}/AnnService/inc/Socket/*.h)
 file(GLOB SERVER_FILES ${SRC_FILES} ${PROJECT_SOURCE_DIR}/AnnService/src/Server/*.cpp ${PROJECT_SOURCE_DIR}/AnnService/src/Socket/*.cpp)
 add_executable (server ${SERVER_FILES} ${SERVER_HDR_FILES})
-target_link_libraries(server ${Boost_LIBRARIES})
+target_link_libraries(server ${Boost_LIBRARIES} ${TBB_LIBRARIES})
 
 file(GLOB CLIENT_HDR_FILES ${HDR_FILES} ${PROJECT_SOURCE_DIR}/AnnService/inc/Client/*.h ${PROJECT_SOURCE_DIR}/AnnService/inc/Socket/*.h)
 file(GLOB CLIENT_FILES ${SRC_FILES} ${PROJECT_SOURCE_DIR}/AnnService/src/Client/*.cpp ${PROJECT_SOURCE_DIR}/AnnService/src/Socket/*.cpp)
 add_executable (client ${CLIENT_FILES} ${CLIENT_HDR_FILES})
-target_link_libraries(client ${Boost_LIBRARIES})
+target_link_libraries(client ${Boost_LIBRARIES} ${TBB_LIBRARIES})
 
 file(GLOB AGG_HDR_FILES ${HDR_FILES} ${PROJECT_SOURCE_DIR}/AnnService/inc/Aggregator/*.h ${PROJECT_SOURCE_DIR}/AnnService/inc/Socket/*.h)
 file(GLOB AGG_FILES ${SRC_FILES} ${PROJECT_SOURCE_DIR}/AnnService/src/Aggregator/*.cpp ${PROJECT_SOURCE_DIR}/AnnService/src/Socket/*.cpp)
 add_executable (aggregator ${AGG_FILES} ${AGG_HDR_FILES})
-target_link_libraries(aggregator ${Boost_LIBRARIES})
+target_link_libraries(aggregator ${Boost_LIBRARIES} ${TBB_LIBRARIES})
 
 file(GLOB BUILDER_HDR_FILES ${HDR_FILES} ${PROJECT_SOURCE_DIR}/AnnService/inc/IndexBuilder/*.h ${PROJECT_SOURCE_DIR}/AnnService/inc/IndexBuilder/VectorSetReaders/*.h)
 file(GLOB BUILDER_FILES ${SRC_FILES} ${PROJECT_SOURCE_DIR}/AnnService/src/IndexBuilder/*.cpp ${PROJECT_SOURCE_DIR}/AnnService/src/IndexBuilder/VectorSetReaders/*.cpp)
 add_executable (indexbuilder ${BUILDER_FILES} ${BUILDER_HDR_FILES})
-target_link_libraries(indexbuilder ${Boost_LIBRARIES})
+target_link_libraries(indexbuilder ${Boost_LIBRARIES} ${TBB_LIBRARIES})
 
 install(TARGETS SPTAGLib SPTAGLibStatic server client aggregator indexbuilder
   RUNTIME DESTINATION bin
   ARCHIVE DESTINATION lib
-  LIBRARY DESTINATION lib)
+  LIBRARY DESTINATION lib)
diff --git a/AnnService/CoreLibrary.vcxproj b/AnnService/CoreLibrary.vcxproj
@@ -131,6 +131,7 @@
     </Link>
   </ItemDefinitionGroup>
   <ItemGroup>
+    <ClInclude Include="inc\Core\Common\FineGrainedLock.h" />
     <ClInclude Include="inc\Core\Common\WorkSpace.h" />
     <ClInclude Include="inc\Core\Common\CommonUtils.h" />
     <ClInclude Include="inc\Core\Common\Dataset.h" />
@@ -171,7 +172,17 @@
     <ClCompile Include="src\Helper\Concurrent.cpp" />
     <ClCompile Include="src\Helper\SimpleIniReader.cpp" />
   </ItemGroup>
+  <ItemGroup>
+    <None Include="packages.config" />
+  </ItemGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
+    <Import Project="..\packages\tbb_oss.9.107.0.0\build\native\tbb_oss.targets" Condition="Exists('..\packages\tbb_oss.9.107.0.0\build\native\tbb_oss.targets')" />
   </ImportGroup>
+  <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
+    <PropertyGroup>
+      <ErrorText>This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them.  For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
+    </PropertyGroup>
+    <Error Condition="!Exists('..\packages\tbb_oss.9.107.0.0\build\native\tbb_oss.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\tbb_oss.9.107.0.0\build\native\tbb_oss.targets'))" />
+  </Target>
 </Project>
diff --git a/AnnService/CoreLibrary.vcxproj.filters b/AnnService/CoreLibrary.vcxproj.filters
@@ -115,6 +115,9 @@
     <ClInclude Include="inc\Core\KDT\ParameterDefinitionList.h">
       <Filter>Header Files\Core\KDT</Filter>
     </ClInclude>
+    <ClInclude Include="inc\Core\Common\FineGrainedLock.h">
+      <Filter>Header Files\Core\Common</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="src\Core\CommonDataStructure.cpp">
@@ -132,9 +135,6 @@
     <ClCompile Include="src\Helper\SimpleIniReader.cpp">
       <Filter>Source Files\Helper</Filter>
     </ClCompile>
-    <ClCompile Include="src\Core\Common.cpp">
-      <Filter>Source Files\Core</Filter>
-    </ClCompile>
     <ClCompile Include="src\Core\VectorSet.cpp">
       <Filter>Source Files\Core</Filter>
     </ClCompile>

diff --git a/AnnService/IndexBuilder.vcxproj b/AnnService/IndexBuilder.vcxproj
@@ -159,6 +159,7 @@
     <Import Project="..\packages\boost_system-vc140.1.67.0.0\build\boost_system-vc140.targets" Condition="Exists('..\packages\boost_system-vc140.1.67.0.0\build\boost_system-vc140.targets')" />
     <Import Project="..\packages\boost_thread-vc140.1.67.0.0\build\boost_thread-vc140.targets" Condition="Exists('..\packages\boost_thread-vc140.1.67.0.0\build\boost_thread-vc140.targets')" />
     <Import Project="..\packages\boost_wserialization-vc140.1.67.0.0\build\boost_wserialization-vc140.targets" Condition="Exists('..\packages\boost_wserialization-vc140.1.67.0.0\build\boost_wserialization-vc140.targets')" />
+    <Import Project="..\packages\tbb_oss.9.107.0.0\build\native\tbb_oss.targets" Condition="Exists('..\packages\tbb_oss.9.107.0.0\build\native\tbb_oss.targets')" />
   </ImportGroup>
   <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
     <PropertyGroup>
@@ -171,5 +172,6 @@
     <Error Condition="!Exists('..\packages\boost_system-vc140.1.67.0.0\build\boost_system-vc140.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\boost_system-vc140.1.67.0.0\build\boost_system-vc140.targets'))" />
     <Error Condition="!Exists('..\packages\boost_thread-vc140.1.67.0.0\build\boost_thread-vc140.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\boost_thread-vc140.1.67.0.0\build\boost_thread-vc140.targets'))" />
     <Error Condition="!Exists('..\packages\boost_wserialization-vc140.1.67.0.0\build\boost_wserialization-vc140.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\boost_wserialization-vc140.1.67.0.0\build\boost_wserialization-vc140.targets'))" />
+    <Error Condition="!Exists('..\packages\tbb_oss.9.107.0.0\build\native\tbb_oss.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\tbb_oss.9.107.0.0\build\native\tbb_oss.targets'))" />
   </Target>
 </Project>
diff --git a/AnnService/Server.vcxproj b/AnnService/Server.vcxproj
@@ -137,6 +137,7 @@
     <Import Project="..\packages\boost_system-vc140.1.67.0.0\build\boost_system-vc140.targets" Condition="Exists('..\packages\boost_system-vc140.1.67.0.0\build\boost_system-vc140.targets')" />
     <Import Project="..\packages\boost_thread-vc140.1.67.0.0\build\boost_thread-vc140.targets" Condition="Exists('..\packages\boost_thread-vc140.1.67.0.0\build\boost_thread-vc140.targets')" />
     <Import Project="..\packages\boost_wserialization-vc140.1.67.0.0\build\boost_wserialization-vc140.targets" Condition="Exists('..\packages\boost_wserialization-vc140.1.67.0.0\build\boost_wserialization-vc140.targets')" />
+    <Import Project="..\packages\tbb_oss.9.107.0.0\build\native\tbb_oss.targets" Condition="Exists('..\packages\tbb_oss.9.107.0.0\build\native\tbb_oss.targets')" />
   </ImportGroup>
   <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
     <PropertyGroup>
@@ -149,5 +150,6 @@
     <Error Condition="!Exists('..\packages\boost_system-vc140.1.67.0.0\build\boost_system-vc140.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\boost_system-vc140.1.67.0.0\build\boost_system-vc140.targets'))" />
     <Error Condition="!Exists('..\packages\boost_thread-vc140.1.67.0.0\build\boost_thread-vc140.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\boost_thread-vc140.1.67.0.0\build\boost_thread-vc140.targets'))" />
     <Error Condition="!Exists('..\packages\boost_wserialization-vc140.1.67.0.0\build\boost_wserialization-vc140.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\boost_wserialization-vc140.1.67.0.0\build\boost_wserialization-vc140.targets'))" />
+    <Error Condition="!Exists('..\packages\tbb_oss.9.107.0.0\build\native\tbb_oss.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\tbb_oss.9.107.0.0\build\native\tbb_oss.targets'))" />
   </Target>
 </Project>
diff --git a/AnnService/inc/Core/BKT/Index.h b/AnnService/inc/Core/BKT/Index.h
@@ -12,11 +12,13 @@
 #include "../Common/Dataset.h"
 #include "../Common/WorkSpace.h"
 #include "../Common/WorkSpacePool.h"
+#include "../Common/FineGrainedLock.h"
+#include "../Common/DataUtils.h"
 
 #include <functional>
-#include <list>
 #include <mutex>
 #include <stack>
+#include <tbb/concurrent_unordered_set.h>
 
 namespace SPTAG
 {
@@ -36,7 +38,7 @@ namespace BKT
         int childStart;
         int childEnd;
 
-        BKTNode(int cid = -1) : centerid(cid), childStart(-1) {}
+        BKTNode(int cid = -1) : centerid(cid), childStart(-1), childEnd(-1) {}
     };
 
     template <typename T>
@@ -119,8 +121,7 @@ namespace BKT
         int m_iDataSize;
         int m_iDataDimension;
         COMMON::Dataset<T> m_pSamples;
-        std::shared_ptr<MetadataSet> m_pMetadata;
-
+
         // BKT structures. 
         int m_iBKTNumber;
         std::vector<int> m_pBKTStart;
@@ -156,7 +157,6 @@ namespace BKT
         char* m_pGraphMemoryFile;
         char* m_pDataPointsMemoryFile;
 
-        int m_iNumberOfThreads;
         DistCalcMethod m_iDistCalcMethod;
         float(*m_fComputeDistance)(const T* pX, const T* pY, int length);
 
@@ -167,8 +167,11 @@ namespace BKT
         int g_iNumberOfInitialDynamicPivots;
         int g_iNumberOfOtherDynamicPivots;
 
+        int m_iNumberOfThreads;
+        std::mutex m_dataAllocLock;
+        COMMON::FineGrainedLock m_dataUpdateLock;
+        tbb::concurrent_unordered_set<int> m_deletedID;
         std::unique_ptr<COMMON::WorkSpacePool> m_workSpacePool;
-
     public:
         Index() : m_iBKTNumber(1),
             m_iBKTKmeansK(32),
@@ -204,95 +207,29 @@ namespace BKT
         int GetFeatureDim() const { return m_pSamples.C(); }
         int GetNumThreads() const { return m_iNumberOfThreads; }
         int GetCurrMaxCheck() const { return m_iMaxCheck; }
+
         DistCalcMethod GetDistCalcMethod() const { return m_iDistCalcMethod; }
         IndexAlgoType GetIndexAlgoType() const { return IndexAlgoType::BKT; }
-        VectorValueType AcceptableQueryValueType() const { return GetEnumValueType<T>(); }
-        void SetMetadata(const std::string& metadataFilePath, const std::string& metadataIndexPath) {
-            m_pMetadata.reset(new FileMetadataSet(metadataFilePath, metadataIndexPath));
-        }
-        ByteArray GetMetadata(IndexType p_vectorID) const {
-            if (nullptr != m_pMetadata)
-            {
-                return m_pMetadata->GetMetadata(p_vectorID);
-            }
-            return ByteArray::c_empty;
-        }
+        VectorValueType GetVectorValueType() const { return GetEnumValueType<T>(); }
 
-        bool BuildIndex();
-        bool BuildIndex(void* p_data, int p_vectorNum, int p_dimension);
-        ErrorCode BuildIndex(std::shared_ptr<VectorSet> p_vectorSet,
-            std::shared_ptr<MetadataSet> p_metadataSet);
+        ErrorCode BuildIndex(const void* p_data, int p_vectorNum, int p_dimension);
+
+        ErrorCode LoadIndex(const std::string& p_folderPath);
+        ErrorCode LoadIndexFromMemory(const std::vector<void*>& p_indexBlobs);
 
-        bool LoadIndex();
-        ErrorCode LoadIndex(const std::string& p_folderPath, const Helper::IniReader& p_configReader);
-
-        bool SaveIndex();
         ErrorCode SaveIndex(const std::string& p_folderPath);
 
-        void SearchIndex(COMMON::QueryResultSet<T> &query, COMMON::WorkSpace &space) const;
-        ErrorCode SearchIndex(QueryResult &query) const;
-
-        void AddNodes(const T* pData, int num, COMMON::WorkSpace &space);
-
+        void SearchIndex(COMMON::QueryResultSet<T> &p_query, COMMON::WorkSpace &p_space, const tbb::concurrent_unordered_set<int> &p_deleted) const;
+        ErrorCode SearchIndex(QueryResult &p_query) const;
+
+        ErrorCode AddIndex(const void* p_vectors, int p_vectorNum, int p_dimension);
+        ErrorCode DeleteIndex(const void* p_vectors, int p_vectorNum);
+        ErrorCode RefineIndex(const std::string& p_folderPath);
+        ErrorCode MergeIndex(const char* p_indexFilePath1, const char* p_indexFilePath2);
+
         ErrorCode SetParameter(const char* p_param, const char* p_value);
         std::string GetParameter(const char* p_param) const;
 
-        // This can be used for both building model files or searching with model files loaded.
-        void SetParameters(std::string dataPointsFile,
-            std::string BKTFile,
-            std::string graphFile,
-            int numBKT,
-            int neighborhoodSize,
-            int kmeansK,
-            int BKTLeafSize,
-            int numSamplesBKT,
-            int numTPTrees,
-            int TPTLeafSize,
-            int maxCheckForRefineGraph,
-            int numThreads,
-            DistCalcMethod distCalcMethod,
-            int cacheSize = -1,
-            int numPoints = -1)
-        {
-            m_sDataPointsFilename = dataPointsFile;
-            m_sBKTFilename = BKTFile;
-            m_sGraphFilename = graphFile;
-            m_iBKTNumber = numBKT;
-            m_iNeighborhoodSize = neighborhoodSize;
-            m_iBKTKmeansK = kmeansK;
-            m_iBKTLeafSize = BKTLeafSize;
-            m_iSamples = numSamplesBKT;
-            m_iTptreeNumber = numTPTrees;
-            m_iTPTLeafSize = TPTLeafSize;
-            m_iMaxCheckForRefineGraph = maxCheckForRefineGraph;
-            m_iNumberOfThreads = numThreads;
-            m_iDistCalcMethod = distCalcMethod;
-            m_fComputeDistance = COMMON::DistanceCalcSelector<T>(m_iDistCalcMethod);
-
-            m_iCacheSize = cacheSize;
-            m_iDebugLoad = numPoints;
-        }
-
-        // Only used for searching with memory mapped model files
-        void SetParameters(char* pDataPointsMemFile,
-            char* pBKTMemFile,
-            char* pGraphMemFile,
-            DistCalcMethod distCalcMethod,
-            int maxCheck,
-            int numBKT,
-            int neighborhoodSize)
-        {
-            m_pDataPointsMemoryFile = pDataPointsMemFile;
-            m_pBKTMemoryFile = pBKTMemFile;
-            m_pGraphMemoryFile = pGraphMemFile;
-            m_iMaxCheck = maxCheck;
-            m_iBKTNumber = numBKT;
-            m_iNeighborhoodSize = neighborhoodSize;
-            m_iNumberOfThreads = 1;
-            m_iDistCalcMethod = distCalcMethod;
-            m_fComputeDistance = COMMON::DistanceCalcSelector<T>(m_iDistCalcMethod);
-        }
-
     private:
         // Functions for loading models from files
         bool LoadDataPoints(std::string sDataPointsFileName);
@@ -307,8 +244,8 @@ namespace BKT
         bool SaveDataPoints(std::string sDataPointsFileName);
 
         // Functions for building balanced kmeans tree
-        void BuildBKT();
-        bool SaveBKT(std::string sBKTFilename) const;
+        void BuildBKT(std::vector<int>& indices, std::vector<int>& newStart, std::vector<BKTNode>& newRoot);
+        bool SaveBKT(std::string sBKTFilename, std::vector<int>& newStart, std::vector<BKTNode>& newRoot) const;
         float KmeansAssign(std::vector<int>& indices, const int first, const int last, KmeansArgs<T>& args, bool updateCenters);
         int KmeansClustering(std::vector<int>& indices, const int first, const int last, KmeansArgs<T>& args);
 

diff --git a/AnnService/inc/Core/Common.h b/AnnService/inc/Core/Common.h
@@ -7,6 +7,7 @@
 #include <string>
 #include <limits>
 #include <vector>
+#include <cmath>
 
 #ifndef _MSC_VER
 #include <sys/stat.h>
@@ -29,6 +30,11 @@ template <class T>
 inline T max(T a, T b) {
     return a > b ? a : b;
 }
+
+#ifndef _rotl
+#define _rotl(x, n) (((x) << (n)) | ((x) >> (32-(n))))
+#endif
+
 #else
 #define WIN32_LEAN_AND_MEAN
 #include <Windows.h>