Add new --append option to IndexBuilder to support adding new items t…

…o an existing index
microsoft · May 30, 2019 · 75f2759 · 75f2759
1 parent a00b58b
commit 75f2759
Show file tree

Hide file tree

Showing 3 changed files with 23 additions and 8 deletions.
diff --git a/AnnService/inc/IndexBuilder/Options.h b/AnnService/inc/IndexBuilder/Options.h
@@ -29,6 +29,8 @@ class BuilderOptions : public Helper::ArgumentsParser
 
     std::string m_vectorDelimiter;
 
+	bool m_append;
+
     SPTAG::VectorValueType m_inputValueType;
 
     std::string m_inputFiles;

diff --git a/AnnService/src/IndexBuilder/Options.cpp b/AnnService/src/IndexBuilder/Options.cpp
@@ -22,7 +22,8 @@ BuilderOptions::BuilderOptions()
     AddRequiredOption(m_inputFiles, "-i", "--input", "Input raw data.");
     AddRequiredOption(m_outputFolder, "-o", "--outputfolder", "Output folder.");
     AddRequiredOption(m_indexAlgoType, "-a", "--algo", "Index Algorithm type.");
-    AddOptionalOption(m_builderConfigFile, "-c", "--config", "Config file for builder.");
+	AddOptionalOption(m_builderConfigFile, "-c", "--config", "Config file for builder.");
+	AddOptionalOption(m_append, "-p", "--append", "Append to existing index.");
 }
 
 

diff --git a/AnnService/src/IndexBuilder/main.cpp b/AnnService/src/IndexBuilder/main.cpp
@@ -55,7 +55,9 @@ int main(int argc, char* argv[])
         indexBuilder->SetParameter(iter.first.c_str(), iter.second.c_str());
     }
 
-    ErrorCode code;
+	std::shared_ptr<VectorSet> p_vectorSet = nullptr;
+	std::shared_ptr<MetadataSet> p_metaSet = nullptr;
+
     if (options->m_inputFiles.find("BIN:") == 0) {
         std::vector<std::string> files = SPTAG::Helper::StrUtils::SplitString(options->m_inputFiles.substr(4), ",");
         std::ifstream inputStream(files[0], std::ifstream::binary);
@@ -71,14 +73,12 @@ int main(int argc, char* argv[])
         char* vecBuf = reinterpret_cast<char*>(vectorSet.Data());
         inputStream.read(vecBuf, totalRecordVectorBytes);
         inputStream.close();
-        std::shared_ptr<VectorSet> p_vectorSet(new BasicVectorSet(vectorSet, options->m_inputValueType, col, row));
+
+        p_vectorSet = std::make_shared<BasicVectorSet>(*new BasicVectorSet(vectorSet, options->m_inputValueType, col, row));
 
-        std::shared_ptr<MetadataSet> p_metaSet = nullptr;
         if (files.size() >= 3) {
             p_metaSet.reset(new FileMetadataSet(files[1], files[2]));
         }
-        code = indexBuilder->BuildIndex(p_vectorSet, p_metaSet);
-        indexBuilder->SaveIndex(options->m_outputFolder);
     }
     else {
         auto vectorReader = IndexBuilder::VectorSetReader::CreateInstance(options);
@@ -87,10 +87,22 @@ int main(int argc, char* argv[])
             fprintf(stderr, "Failed to read input file.\n");
             exit(1);
         }
-        code = indexBuilder->BuildIndex(vectorReader->GetVectorSet(), vectorReader->GetMetadataSet());
-        indexBuilder->SaveIndex(options->m_outputFolder);
+
+		p_vectorSet = vectorReader->GetVectorSet();
+		p_metaSet = vectorReader->GetMetadataSet();
     }
 
+	ErrorCode code;
+	std::shared_ptr<SPTAG::VectorIndex> vecIndex;
+	if (options->m_append && ErrorCode::Success == indexBuilder->LoadIndex(options->m_outputFolder, vecIndex) && nullptr != vecIndex) {
+		code = vecIndex->AddIndex(p_vectorSet, p_metaSet);
+		indexBuilder = vecIndex;
+	}
+	else {
+		code = indexBuilder->BuildIndex(p_vectorSet, p_metaSet);
+	}
+	indexBuilder->SaveIndex(options->m_outputFolder);
+
     if (ErrorCode::Success != code)
     {
         fprintf(stderr, "Failed to build index.\n");