change code structure and try to clean wrapper definitions (#8)

* remove dup code

* Update Readme.md

* Fix DataSet GNU compile fail bug

* fix GNU Windows align alloc bugs

* add copyright in each file

* fix copy right in dataset

* change kdt distance judgement

* change code structure and add more wrappers

* Update docs

* fix search result

* change IndexBuilder to support binary input data

* temp remove java related projects

* remove javaclient and javacore from the windows build

* Fix SetData issue

.gitignore

@@ -72,3 +72,19 @@ dkms.conf
 /x64/Debug
 /packages
 /Search/Search.vcxproj.user
+/AnnService/IndexSearcher.vcxproj.user
+/Wrappers/inc/SWIGTYPE_p_RemoteSearchResult.java
+/Wrappers/inc/SWIGTYPE_p_QueryResult.java
+/Wrappers/inc/SPTAGJNI.java
+/Wrappers/inc/SPTAGClientJNI.java
+/Wrappers/inc/SPTAGClient.py
+/Wrappers/inc/SPTAGClient.java
+/Wrappers/inc/SPTAG.py
+/Wrappers/inc/SPTAG.java
+/Wrappers/inc/CoreInterface_pwrap.cpp
+/Wrappers/inc/CoreInterface_jwrap.cpp
+/Wrappers/inc/ClientInterface_pwrap.cpp
+/Wrappers/inc/ClientInterface_jwrap.cpp
+/Wrappers/inc/AnnIndex.java
+/Wrappers/inc/AnnClient.java
+/AnnService.users - Copy.props

AnnService.users.props

@@ -11,6 +11,8 @@
   <PropertyGroup Label="UserMacros">
 	<OutAppDir>$(SolutionDir)\$(Platform)\$(Configuration)\</OutAppDir>
     <OutLibDir>$(SolutionDir)\$(Platform)\$(Configuration)\</OutLibDir>
+	<JavaIncDir></JavaIncDir>
+	<JavaLib></JavaLib>
   </PropertyGroup>
 </Project>
 

AnnService/CMakeLists.txt

@@ -7,6 +7,7 @@ file(GLOB SRC_FILES ${PROJECT_SOURCE_DIR}/AnnService/src/Core/*.cpp ${PROJECT_SO
 include_directories(${PROJECT_SOURCE_DIR}/AnnService)
 
 add_library (SPTAGLib SHARED ${SRC_FILES} ${HDR_FILES})
+target_link_libraries (SPTAGLib ${TBB_LIBRARIES})
 add_library (SPTAGLibStatic STATIC ${SRC_FILES} ${HDR_FILES})
 set_target_properties(SPTAGLibStatic PROPERTIES OUTPUT_NAME SPTAGLib)
 
@@ -30,7 +31,11 @@ file(GLOB BUILDER_FILES ${SRC_FILES} ${PROJECT_SOURCE_DIR}/AnnService/src/IndexB
 add_executable (indexbuilder ${BUILDER_FILES} ${BUILDER_HDR_FILES})
 target_link_libraries(indexbuilder ${Boost_LIBRARIES} ${TBB_LIBRARIES})
 
-install(TARGETS SPTAGLib SPTAGLibStatic server client aggregator indexbuilder
+file(GLOB SEARCHER_FILES ${SRC_FILES} ${PROJECT_SOURCE_DIR}/AnnService/src/IndexSearcher/*.cpp)
+add_executable (indexsearcher ${SEARCHER_FILES} ${HDR_FILES})
+target_link_libraries(indexsearcher ${Boost_LIBRARIES} ${TBB_LIBRARIES})
+
+install(TARGETS SPTAGLib SPTAGLibStatic server client aggregator indexbuilder indexsearcher
   RUNTIME DESTINATION bin
   ARCHIVE DESTINATION lib
   LIBRARY DESTINATION lib)

Search/Search.vcxproj → AnnService/IndexSearcher.vcxproj

@@ -21,9 +21,9 @@
   <PropertyGroup Label="Globals">
     <VCProjectVersion>15.0</VCProjectVersion>
     <ProjectGuid>{97615D3B-9FA0-469E-B229-95A91A5087E0}</ProjectGuid>
-    <RootNamespace>Search</RootNamespace>
+    <RootNamespace>IndexSearcher</RootNamespace>
     <WindowsTargetPlatformVersion>8.1</WindowsTargetPlatformVersion>
-    <ProjectName>Search</ProjectName>
+    <ProjectName>IndexSearcher</ProjectName>
   </PropertyGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
   <Import Project="$(SolutionDir)\AnnService.users.props" />
@@ -136,7 +136,7 @@
     </Link>
   </ItemDefinitionGroup>
   <ItemGroup>
-    <ClCompile Include="main.cpp" />
+    <ClCompile Include="src\IndexSearcher\main.cpp" />
   </ItemGroup>
   <ItemGroup>
     <None Include="packages.config">

Search/Search.vcxproj.filters → AnnService/IndexSearcher.vcxproj.filters

@@ -15,7 +15,7 @@
     </Filter>
   </ItemGroup>
   <ItemGroup>
-    <ClCompile Include="main.cpp">
+    <ClCompile Include="src\IndexSearcher\main.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
   </ItemGroup>

AnnService/inc/Core/BKT/ParameterDefinitionList.h

@@ -19,8 +19,8 @@ DefineBKTParameter(m_pGraph.m_iTPTLeafSize, int, 2000L, "TPTLeafSize")
 DefineBKTParameter(m_pGraph.m_numTopDimensionTPTSplit, int, 5L, "NumTopDimensionTpTreeSplit")
 
 DefineBKTParameter(m_pGraph.m_iNeighborhoodSize, int, 32L, "NeighborhoodSize")
-DefineBKTParameter(m_pGraph.m_iNeighborhoodScale, int, 16L, "GraphNeighborhoodScale")
-DefineBKTParameter(m_pGraph.m_iCEFScale, int, 4L, "GraphCEFScale")
+DefineBKTParameter(m_pGraph.m_iNeighborhoodScale, int, 2L, "GraphNeighborhoodScale")
+DefineBKTParameter(m_pGraph.m_iCEFScale, int, 2L, "GraphCEFScale")
 DefineBKTParameter(m_pGraph.m_iRefineIter, int, 0L, "RefineIterations")
 DefineBKTParameter(m_pGraph.m_iCEF, int, 1000L, "CEF")
 DefineBKTParameter(m_pGraph.m_iMaxCheckForRefineGraph, int, 10000L, "MaxCheckForRefineGraph")

AnnService/inc/Core/Common/NeighborhoodGraph.h

@@ -121,6 +121,7 @@ namespace SPTAG
                 for (int i = 0; i < m_iGraphSize; i++)
                 {
                     RefineNode<T>(index, i, false);
+					if (i % 1000 == 0) std::cout << "\rRefine 1 " << (i * 100 / m_iGraphSize) << "%";
                 }
                 std::cout << "Refine RNG, graph acc:" << GraphAccuracyEstimation(index, 100, idmap) << std::endl;
 
@@ -132,6 +133,7 @@ namespace SPTAG
                 for (int i = 0; i < m_iGraphSize; i++)
                 {
                     RefineNode<T>(index, i, false);
+					if (i % 1000 == 0) std::cout << "\rRefine 2 " << (i * 100 / m_iGraphSize) << "%";
                 }
                 std::cout << "Refine RNG, graph acc:" << GraphAccuracyEstimation(index, 100, idmap) << std::endl;
 

AnnService/inc/Core/CommonDataStructure.h

@@ -32,7 +32,9 @@ class ByteArray
 
     std::uint8_t* Data() const;
 
-    std::size_t Length() const;
+	std::size_t Length() const;
+
+	void SetData(std::uint8_t* p_array, std::size_t p_length);
 
     std::shared_ptr<std::uint8_t> DataHolder() const;
 

AnnService/inc/Core/KDT/ParameterDefinitionList.h

@@ -17,8 +17,8 @@ DefineKDTParameter(m_pGraph.m_iTPTLeafSize, int, 2000L, "TPTLeafSize")
 DefineKDTParameter(m_pGraph.m_numTopDimensionTPTSplit, int, 5L, "NumTopDimensionTPTSplit")
 
 DefineKDTParameter(m_pGraph.m_iNeighborhoodSize, int, 32L, "NeighborhoodSize")
-DefineKDTParameter(m_pGraph.m_iNeighborhoodScale, int, 16L, "GraphNeighborhoodScale")
-DefineKDTParameter(m_pGraph.m_iCEFScale, int, 4L, "GraphCEFScale")
+DefineKDTParameter(m_pGraph.m_iNeighborhoodScale, int, 2L, "GraphNeighborhoodScale")
+DefineKDTParameter(m_pGraph.m_iCEFScale, int, 2L, "GraphCEFScale")
 DefineKDTParameter(m_pGraph.m_iRefineIter, int, 0L, "RefineIterations")
 DefineKDTParameter(m_pGraph.m_iCEF, int, 1000L, "CEF")
 DefineKDTParameter(m_pGraph.m_iMaxCheckForRefineGraph, int, 10000L, "MaxCheckForRefineGraph")

AnnService/src/Core/CommonDataStructure.cpp

@@ -104,7 +104,15 @@ ByteArray::Data() const
 std::size_t
 ByteArray::Length() const
 {
-    return m_length;
+	return m_length;
+}
+
+
+void
+ByteArray::SetData(std::uint8_t* p_array, std::size_t p_length)
+{
+    m_data = p_array;
+	m_length = p_length;
 }
 
 

AnnService/src/IndexBuilder/main.cpp

@@ -57,8 +57,8 @@ int main(int argc, char* argv[])
 
     ErrorCode code;
     if (options->m_inputFiles.find("BIN:") == 0) {
-        options->m_inputFiles = options->m_inputFiles.substr(4);
-        std::ifstream inputStream(options->m_inputFiles, std::ifstream::binary);
+        std::vector<std::string> files = SPTAG::Helper::StrUtils::SplitString(options->m_inputFiles.substr(4), ",");
+        std::ifstream inputStream(files[0], std::ifstream::binary);
         if (!inputStream.is_open()) {
             fprintf(stderr, "Failed to read input file.\n");
             exit(1);
@@ -71,9 +71,13 @@ int main(int argc, char* argv[])
         char* vecBuf = reinterpret_cast<char*>(vectorSet.Data());
         inputStream.read(vecBuf, totalRecordVectorBytes);
         inputStream.close();
-
         std::shared_ptr<VectorSet> p_vectorSet(new BasicVectorSet(vectorSet, options->m_inputValueType, col, row));
-        code = indexBuilder->BuildIndex(p_vectorSet, nullptr);
+
+        std::shared_ptr<MetadataSet> p_metaSet = nullptr;
+        if (files.size() >= 3) {
+            p_metaSet.reset(new FileMetadataSet(files[1], files[2]));
+        }
+        code = indexBuilder->BuildIndex(p_vectorSet, p_metaSet);
         indexBuilder->SaveIndex(options->m_outputFolder);
     }
     else {

CMakeLists.txt

@@ -86,15 +86,29 @@ else()
     message (FATAL_ERROR "Could not find Boost 1.67!")
 endif()
 
-find_library(TBB_LIBRARIES libtbb${CMAKE_SHARED_LIBRARY_SUFFIX})
-if (TBB_LIBRARIES)
+if (WIN32)
+    if (${CMAKE_SIZEOF_VOID_P} EQUAL "8")
+        set (TBB_LIBRARY_SUFFIX "lib/intel64/vc14")
+    else()
+        set (TBB_LIBRARY_SUFFIX "lib/ia32/vc14")
+    endif()
+
+    find_path(TBB_INCLUDE_DIRS tbb/tbb.h HINTS ${TBB_DIR} "C:/Program Files/Intel/TBB" PATH_SUFFIXES include)
+    find_library(TBB_LIBRARIES tbb${CMAKE_STATIC_LIBRARY_SUFFIX} HINTS ${TBB_DIR} "C:/Program Files/Intel/TBB" PATH_SUFFIXES ${TBB_LIBRARY_SUFFIX})
+else()
+    find_path(TBB_INCLUDE_DIRS tbb/tbb.h HINTS ${TBB_DIR} "/usr/" PATH_SUFFIXES include)
+    find_library(TBB_LIBRARIES libtbb${CMAKE_SHARED_LIBRARY_SUFFIX} HINTS ${TBB_DIR} "/usr/")
+endif()
+
+if (TBB_INCLUDE_DIRS AND TBB_LIBRARIES)
+    include_directories (${TBB_INCLUDE_DIRS})
     message (STATUS "Found TBB.")
-    message (STATUS "Library: ${TBB_LIBRARIES}")
+    message (STATUS "Include Path:" ${TBB_INCLUDE_DIRS})
+    message (STATUS "Library:" ${TBB_LIBRARIES})
 else()
     message (FATAL_ERROR "Could not find TBB!")
 endif()
 
 add_subdirectory (AnnService)
-add_subdirectory (PythonWrapper)
+add_subdirectory (Wrappers)
 add_subdirectory (Test)
-add_subdirectory (Search)

README.md

@@ -43,6 +43,7 @@ The searches in the trees and the graph are iteratively conducted.
 * swig >= 3.0
 * cmake >= 3.12.0
 * boost >= 1.67.0
+* tbb >= 4.2
 
 ### **Install**
 

SPTAG.sln

@@ -11,7 +11,7 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "Server", "AnnService\Server
 		{C2BC5FDE-C853-4F3D-B7E4-2C9B5524DDF9} = {C2BC5FDE-C853-4F3D-B7E4-2C9B5524DDF9}
 	EndProjectSection
 EndProject
-Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "PythonCore", "PythonWrapper\PythonCore.vcxproj", "{AF31947C-0495-42FE-A1AD-8F0DA2A679C7}"
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "PythonCore", "Wrappers\PythonCore.vcxproj", "{AF31947C-0495-42FE-A1AD-8F0DA2A679C7}"
 	ProjectSection(ProjectDependencies) = postProject
 		{C2BC5FDE-C853-4F3D-B7E4-2C9B5524DDF9} = {C2BC5FDE-C853-4F3D-B7E4-2C9B5524DDF9}
 	EndProjectSection
@@ -33,7 +33,7 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "Aggregator", "AnnService\Ag
 		{C2BC5FDE-C853-4F3D-B7E4-2C9B5524DDF9} = {C2BC5FDE-C853-4F3D-B7E4-2C9B5524DDF9}
 	EndProjectSection
 EndProject
-Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "PythonClient", "PythonWrapper\PythonClient.vcxproj", "{9B014CF6-E3FB-4BD4-B3B1-D26297BB31AA}"
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "PythonClient", "Wrappers\PythonClient.vcxproj", "{9B014CF6-E3FB-4BD4-B3B1-D26297BB31AA}"
 	ProjectSection(ProjectDependencies) = postProject
 		{F9A72303-6381-4C80-86FF-606A2F6F7B96} = {F9A72303-6381-4C80-86FF-606A2F6F7B96}
 		{C2BC5FDE-C853-4F3D-B7E4-2C9B5524DDF9} = {C2BC5FDE-C853-4F3D-B7E4-2C9B5524DDF9}
@@ -44,9 +44,27 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "IndexBuilder", "AnnService\
 		{C2BC5FDE-C853-4F3D-B7E4-2C9B5524DDF9} = {C2BC5FDE-C853-4F3D-B7E4-2C9B5524DDF9}
 	EndProjectSection
 EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "IndexSearcher", "AnnService\IndexSearcher.vcxproj", "{97615D3B-9FA0-469E-B229-95A91A5087E0}"
+	ProjectSection(ProjectDependencies) = postProject
+		{C2BC5FDE-C853-4F3D-B7E4-2C9B5524DDF9} = {C2BC5FDE-C853-4F3D-B7E4-2C9B5524DDF9}
+	EndProjectSection
+EndProject
 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "Test", "Test\Test.vcxproj", "{29A25655-CCF2-47F8-8BC8-DFE1B5CF993C}"
+	ProjectSection(ProjectDependencies) = postProject
+		{F9A72303-6381-4C80-86FF-606A2F6F7B96} = {F9A72303-6381-4C80-86FF-606A2F6F7B96}
+		{C2BC5FDE-C853-4F3D-B7E4-2C9B5524DDF9} = {C2BC5FDE-C853-4F3D-B7E4-2C9B5524DDF9}
+	EndProjectSection
 EndProject
-Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "Search", "Search\Search.vcxproj", "{97615D3B-9FA0-469E-B229-95A91A5087E0}"
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "JavaCore", "Wrappers\JavaCore.vcxproj", "{93FEB26B-965E-4157-8BE5-052F5CA112BB}"
+	ProjectSection(ProjectDependencies) = postProject
+		{C2BC5FDE-C853-4F3D-B7E4-2C9B5524DDF9} = {C2BC5FDE-C853-4F3D-B7E4-2C9B5524DDF9}
+	EndProjectSection
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "JavaClient", "Wrappers\JavaClient.vcxproj", "{8866BF98-AA2E-450F-9F33-083E007CCA74}"
+	ProjectSection(ProjectDependencies) = postProject
+		{F9A72303-6381-4C80-86FF-606A2F6F7B96} = {F9A72303-6381-4C80-86FF-606A2F6F7B96}
+		{C2BC5FDE-C853-4F3D-B7E4-2C9B5524DDF9} = {C2BC5FDE-C853-4F3D-B7E4-2C9B5524DDF9}
+	EndProjectSection
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -120,14 +138,6 @@ Global
 		{F492F794-E78B-4B1F-A556-5E045B9163D5}.Release|x64.Build.0 = Release|x64
 		{F492F794-E78B-4B1F-A556-5E045B9163D5}.Release|x86.ActiveCfg = Release|Win32
 		{F492F794-E78B-4B1F-A556-5E045B9163D5}.Release|x86.Build.0 = Release|Win32
-		{29A25655-CCF2-47F8-8BC8-DFE1B5CF993C}.Debug|x64.ActiveCfg = Debug|x64
-		{29A25655-CCF2-47F8-8BC8-DFE1B5CF993C}.Debug|x64.Build.0 = Debug|x64
-		{29A25655-CCF2-47F8-8BC8-DFE1B5CF993C}.Debug|x86.ActiveCfg = Debug|Win32
-		{29A25655-CCF2-47F8-8BC8-DFE1B5CF993C}.Debug|x86.Build.0 = Debug|Win32
-		{29A25655-CCF2-47F8-8BC8-DFE1B5CF993C}.Release|x64.ActiveCfg = Release|x64
-		{29A25655-CCF2-47F8-8BC8-DFE1B5CF993C}.Release|x64.Build.0 = Release|x64
-		{29A25655-CCF2-47F8-8BC8-DFE1B5CF993C}.Release|x86.ActiveCfg = Release|Win32
-		{29A25655-CCF2-47F8-8BC8-DFE1B5CF993C}.Release|x86.Build.0 = Release|Win32
 		{97615D3B-9FA0-469E-B229-95A91A5087E0}.Debug|x64.ActiveCfg = Debug|x64
 		{97615D3B-9FA0-469E-B229-95A91A5087E0}.Debug|x64.Build.0 = Debug|x64
 		{97615D3B-9FA0-469E-B229-95A91A5087E0}.Debug|x86.ActiveCfg = Debug|Win32
@@ -136,6 +146,22 @@ Global
 		{97615D3B-9FA0-469E-B229-95A91A5087E0}.Release|x64.Build.0 = Release|x64
 		{97615D3B-9FA0-469E-B229-95A91A5087E0}.Release|x86.ActiveCfg = Release|Win32
 		{97615D3B-9FA0-469E-B229-95A91A5087E0}.Release|x86.Build.0 = Release|Win32
+		{29A25655-CCF2-47F8-8BC8-DFE1B5CF993C}.Debug|x64.ActiveCfg = Debug|x64
+		{29A25655-CCF2-47F8-8BC8-DFE1B5CF993C}.Debug|x64.Build.0 = Debug|x64
+		{29A25655-CCF2-47F8-8BC8-DFE1B5CF993C}.Debug|x86.ActiveCfg = Debug|Win32
+		{29A25655-CCF2-47F8-8BC8-DFE1B5CF993C}.Debug|x86.Build.0 = Debug|Win32
+		{29A25655-CCF2-47F8-8BC8-DFE1B5CF993C}.Release|x64.ActiveCfg = Release|x64
+		{29A25655-CCF2-47F8-8BC8-DFE1B5CF993C}.Release|x64.Build.0 = Release|x64
+		{29A25655-CCF2-47F8-8BC8-DFE1B5CF993C}.Release|x86.ActiveCfg = Release|Win32
+		{29A25655-CCF2-47F8-8BC8-DFE1B5CF993C}.Release|x86.Build.0 = Release|Win32
+		{93FEB26B-965E-4157-8BE5-052F5CA112BB}.Debug|x64.ActiveCfg = Debug|x64
+		{93FEB26B-965E-4157-8BE5-052F5CA112BB}.Debug|x86.ActiveCfg = Debug|Win32
+		{93FEB26B-965E-4157-8BE5-052F5CA112BB}.Release|x64.ActiveCfg = Release|x64
+		{93FEB26B-965E-4157-8BE5-052F5CA112BB}.Release|x86.ActiveCfg = Release|Win32
+		{8866BF98-AA2E-450F-9F33-083E007CCA74}.Debug|x64.ActiveCfg = Debug|x64
+		{8866BF98-AA2E-450F-9F33-083E007CCA74}.Debug|x86.ActiveCfg = Debug|Win32
+		{8866BF98-AA2E-450F-9F33-083E007CCA74}.Release|x64.ActiveCfg = Release|x64
+		{8866BF98-AA2E-450F-9F33-083E007CCA74}.Release|x86.ActiveCfg = Release|Win32
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE