From 92fb41bac5e9488e81a9493ecac9a12304ed6428 Mon Sep 17 00:00:00 2001 From: Thejas-bhat Date: Wed, 25 Oct 2023 11:49:26 +0530 Subject: [PATCH 1/5] draft implementation bufIOWriter over uint8_t* buf --- c_api/index_io_c_ex.cpp | 10 ++++++++++ faiss/impl/io.cpp | 34 ++++++++++++++++++++++++++++++++++ faiss/impl/io.h | 15 +++++++++++++++ 3 files changed, 59 insertions(+) diff --git a/c_api/index_io_c_ex.cpp b/c_api/index_io_c_ex.cpp index 03525c0a37..bba43de781 100644 --- a/c_api/index_io_c_ex.cpp +++ b/c_api/index_io_c_ex.cpp @@ -31,10 +31,20 @@ int faiss_write_index_buf(const FaissIndex* idx, int* size, unsigned char** buf) CATCH_AND_HANDLE } +int checksum(const unsigned char* buf, int size) { + + for (int i = 0; i < size; i++) { + + } + + return 0; +} + int faiss_read_index_buf(const unsigned char* buf, int size, int io_flags, FaissIndex** p_out) { try { faiss::VectorIOReader reader; reader.data.assign(buf, buf + size); + printf("the error while reading index from ptr %p of size %d\n", buf, size); auto index = faiss::read_index(&reader, io_flags); *p_out = reinterpret_cast(index); } diff --git a/faiss/impl/io.cpp b/faiss/impl/io.cpp index 3837525138..64c98671f3 100644 --- a/faiss/impl/io.cpp +++ b/faiss/impl/io.cpp @@ -55,6 +55,40 @@ size_t VectorIOReader::operator()(void* ptr, size_t size, size_t nitems) { return nitems; } +/*********************************************************************** + * IO Buffer + ***********************************************************************/ + +// size_t BufIOWriter::operator()(const void* ptr, size_t size, size_t nitems) { +// size_t bytes = size * nitems; +// if (bytes > 0) { +// size_t o = buf_size; +// // data.resize(o + bytes); +// // memcpy(&data[o], ptr, size * nitems); +// buf = (uint8_t*) realloc(buf, o + bytes); +// memcpy(&buf[o], ptr, size * nitems); +// } +// return nitems; +// } + +size_t BufIOReader::operator()(void* ptr, size_t size, size_t nitems) { + if (rp >= buf_size) + return 0; + size_t nremain = (buf_size - rp) / size; + if (nremain < nitems) + nitems = nremain; + if (size * nitems > 0) { + memcpy(ptr, &buf[rp], size * nitems); + rp += size * nitems; + } + + return nitems; +} + +BufIOReader::~BufIOReader() { + free(buf); +} + /*********************************************************************** * IO File ***********************************************************************/ diff --git a/faiss/impl/io.h b/faiss/impl/io.h index 8d0605a5a6..7ca9617965 100644 --- a/faiss/impl/io.h +++ b/faiss/impl/io.h @@ -61,6 +61,21 @@ struct VectorIOWriter : IOWriter { size_t operator()(const void* ptr, size_t size, size_t nitems) override; }; +// struct BufIOWriter : IOWriter { +// uint8_t* buf; +// size_t buf_size; +// size_t operator()(const void* ptr, size_t size, size_t nitems) override; +// ~BufIOWriter() override; +// } + +struct BufIOReader : IOReader { + uint8_t* buf; + size_t rp = 0; + size_t buf_size; + size_t operator()(void* ptr, size_t size, size_t nitems) override; + ~BufIOReader() override; +}; + struct FileIOReader : IOReader { FILE* f = nullptr; bool need_close = false; From da897c850cc46fae777c3c397b584de0f98ccdb0 Mon Sep 17 00:00:00 2001 From: Thejas-bhat Date: Wed, 29 Nov 2023 11:29:41 +0530 Subject: [PATCH 2/5] code cleanup --- c_api/index_io_c_ex.cpp | 18 ++++-------------- faiss/impl/io.cpp | 15 ++------------- faiss/impl/io.h | 9 +-------- 3 files changed, 7 insertions(+), 35 deletions(-) diff --git a/c_api/index_io_c_ex.cpp b/c_api/index_io_c_ex.cpp index bba43de781..22fd748e9d 100644 --- a/c_api/index_io_c_ex.cpp +++ b/c_api/index_io_c_ex.cpp @@ -20,7 +20,6 @@ using faiss::IndexBinary; int faiss_write_index_buf(const FaissIndex* idx, int* size, unsigned char** buf) { try { faiss::VectorIOWriter writer; - faiss::write_index(reinterpret_cast(idx), &writer); unsigned char* tempBuf = (unsigned char*)malloc((writer.data.size()) * sizeof(uint8_t)); std::copy(writer.data.begin(), writer.data.end(), tempBuf); @@ -31,20 +30,11 @@ int faiss_write_index_buf(const FaissIndex* idx, int* size, unsigned char** buf) CATCH_AND_HANDLE } -int checksum(const unsigned char* buf, int size) { - - for (int i = 0; i < size; i++) { - - } - - return 0; -} - -int faiss_read_index_buf(const unsigned char* buf, int size, int io_flags, FaissIndex** p_out) { +int faiss_read_index_buf(const uint8_t* buf, int size, int io_flags, FaissIndex** p_out) { try { - faiss::VectorIOReader reader; - reader.data.assign(buf, buf + size); - printf("the error while reading index from ptr %p of size %d\n", buf, size); + faiss::BufIOReader reader; + reader.buf = buf; + reader.buf_size = size; auto index = faiss::read_index(&reader, io_flags); *p_out = reinterpret_cast(index); } diff --git a/faiss/impl/io.cpp b/faiss/impl/io.cpp index 64c98671f3..1cc6c5eaeb 100644 --- a/faiss/impl/io.cpp +++ b/faiss/impl/io.cpp @@ -37,6 +37,7 @@ size_t VectorIOWriter::operator()(const void* ptr, size_t size, size_t nitems) { if (bytes > 0) { size_t o = data.size(); data.resize(o + bytes); + // printf("performing a memcpy ptr %p data %p\n", ptr, &data[0]); memcpy(&data[o], ptr, size * nitems); } return nitems; @@ -59,18 +60,6 @@ size_t VectorIOReader::operator()(void* ptr, size_t size, size_t nitems) { * IO Buffer ***********************************************************************/ -// size_t BufIOWriter::operator()(const void* ptr, size_t size, size_t nitems) { -// size_t bytes = size * nitems; -// if (bytes > 0) { -// size_t o = buf_size; -// // data.resize(o + bytes); -// // memcpy(&data[o], ptr, size * nitems); -// buf = (uint8_t*) realloc(buf, o + bytes); -// memcpy(&buf[o], ptr, size * nitems); -// } -// return nitems; -// } - size_t BufIOReader::operator()(void* ptr, size_t size, size_t nitems) { if (rp >= buf_size) return 0; @@ -86,7 +75,7 @@ size_t BufIOReader::operator()(void* ptr, size_t size, size_t nitems) { } BufIOReader::~BufIOReader() { - free(buf); + buf = NULL; } /*********************************************************************** diff --git a/faiss/impl/io.h b/faiss/impl/io.h index 7ca9617965..bcc97f5ab9 100644 --- a/faiss/impl/io.h +++ b/faiss/impl/io.h @@ -61,15 +61,8 @@ struct VectorIOWriter : IOWriter { size_t operator()(const void* ptr, size_t size, size_t nitems) override; }; -// struct BufIOWriter : IOWriter { -// uint8_t* buf; -// size_t buf_size; -// size_t operator()(const void* ptr, size_t size, size_t nitems) override; -// ~BufIOWriter() override; -// } - struct BufIOReader : IOReader { - uint8_t* buf; + const uint8_t* buf; size_t rp = 0; size_t buf_size; size_t operator()(void* ptr, size_t size, size_t nitems) override; From dbb2aa9a68e891097e46388e2e71d65656305145 Mon Sep 17 00:00:00 2001 From: Thejas-bhat Date: Wed, 29 Nov 2023 14:17:22 +0530 Subject: [PATCH 3/5] implementation for writing using a ptr instead of vector - BufIOWriter --- c_api/index_io_c_ex.cpp | 12 ++++++------ faiss/impl/io.cpp | 26 +++++++++++++++++++++++++- faiss/impl/io.h | 10 ++++++++++ 3 files changed, 41 insertions(+), 7 deletions(-) diff --git a/c_api/index_io_c_ex.cpp b/c_api/index_io_c_ex.cpp index 22fd748e9d..d7279df54c 100644 --- a/c_api/index_io_c_ex.cpp +++ b/c_api/index_io_c_ex.cpp @@ -19,13 +19,13 @@ using faiss::IndexBinary; int faiss_write_index_buf(const FaissIndex* idx, int* size, unsigned char** buf) { try { - faiss::VectorIOWriter writer; + faiss::BufIOWriter writer; + + writer.buf = (uint8_t*)malloc(writer.buf_cap * sizeof(uint8_t)); faiss::write_index(reinterpret_cast(idx), &writer); - unsigned char* tempBuf = (unsigned char*)malloc((writer.data.size()) * sizeof(uint8_t)); - std::copy(writer.data.begin(), writer.data.end(), tempBuf); - *buf = tempBuf; - *size = writer.data.size(); - writer.data.clear(); + *buf = writer.buf; + *size = writer.buf_size; + writer.buf = NULL; } CATCH_AND_HANDLE } diff --git a/faiss/impl/io.cpp b/faiss/impl/io.cpp index 1cc6c5eaeb..462547c573 100644 --- a/faiss/impl/io.cpp +++ b/faiss/impl/io.cpp @@ -37,7 +37,6 @@ size_t VectorIOWriter::operator()(const void* ptr, size_t size, size_t nitems) { if (bytes > 0) { size_t o = data.size(); data.resize(o + bytes); - // printf("performing a memcpy ptr %p data %p\n", ptr, &data[0]); memcpy(&data[o], ptr, size * nitems); } return nitems; @@ -60,6 +59,31 @@ size_t VectorIOReader::operator()(void* ptr, size_t size, size_t nitems) { * IO Buffer ***********************************************************************/ +size_t BufIOWriter::operator()(const void* ptr, size_t size, size_t nitems) { + size_t bytes = size * nitems; + if (bytes > 0) { + size_t o = buf_size; + bool growing = false; + if (buf_cap == 0) { + buf_cap = o + bytes; + growing = true; + } + while(o + bytes > buf_cap){ + buf_cap = buf_cap * 2; + growing = true; + } + if (growing){ + buf = (uint8_t*) realloc(buf, buf_cap); + } + + memcpy(&buf[o], ptr, size * nitems); + buf_size += (size*nitems); + } + return nitems; +} + +BufIOWriter::~BufIOWriter() {} + size_t BufIOReader::operator()(void* ptr, size_t size, size_t nitems) { if (rp >= buf_size) return 0; diff --git a/faiss/impl/io.h b/faiss/impl/io.h index bcc97f5ab9..e1498c48c1 100644 --- a/faiss/impl/io.h +++ b/faiss/impl/io.h @@ -61,6 +61,16 @@ struct VectorIOWriter : IOWriter { size_t operator()(const void* ptr, size_t size, size_t nitems) override; }; +struct BufIOWriter : IOWriter { + uint8_t* buf; + size_t buf_size = 0; + // default it to zero. the capacity is to be initialized when + // the read operation is performed for the first time with a chunk of data. + size_t buf_cap = 0; + size_t operator()(const void* ptr, size_t size, size_t nitems) override; + ~BufIOWriter() override; +}; + struct BufIOReader : IOReader { const uint8_t* buf; size_t rp = 0; From b27747d1d899b42adaef793861dd70a553eec2c1 Mon Sep 17 00:00:00 2001 From: Thejas-bhat Date: Thu, 30 Nov 2023 15:41:39 +0530 Subject: [PATCH 4/5] revert BufIOWriter implementation --- c_api/index_io_c_ex.cpp | 12 ++++++------ faiss/impl/io.cpp | 25 ------------------------- faiss/impl/io.h | 10 ---------- 3 files changed, 6 insertions(+), 41 deletions(-) diff --git a/c_api/index_io_c_ex.cpp b/c_api/index_io_c_ex.cpp index d7279df54c..22fd748e9d 100644 --- a/c_api/index_io_c_ex.cpp +++ b/c_api/index_io_c_ex.cpp @@ -19,13 +19,13 @@ using faiss::IndexBinary; int faiss_write_index_buf(const FaissIndex* idx, int* size, unsigned char** buf) { try { - faiss::BufIOWriter writer; - - writer.buf = (uint8_t*)malloc(writer.buf_cap * sizeof(uint8_t)); + faiss::VectorIOWriter writer; faiss::write_index(reinterpret_cast(idx), &writer); - *buf = writer.buf; - *size = writer.buf_size; - writer.buf = NULL; + unsigned char* tempBuf = (unsigned char*)malloc((writer.data.size()) * sizeof(uint8_t)); + std::copy(writer.data.begin(), writer.data.end(), tempBuf); + *buf = tempBuf; + *size = writer.data.size(); + writer.data.clear(); } CATCH_AND_HANDLE } diff --git a/faiss/impl/io.cpp b/faiss/impl/io.cpp index 462547c573..ef5e47af9b 100644 --- a/faiss/impl/io.cpp +++ b/faiss/impl/io.cpp @@ -59,31 +59,6 @@ size_t VectorIOReader::operator()(void* ptr, size_t size, size_t nitems) { * IO Buffer ***********************************************************************/ -size_t BufIOWriter::operator()(const void* ptr, size_t size, size_t nitems) { - size_t bytes = size * nitems; - if (bytes > 0) { - size_t o = buf_size; - bool growing = false; - if (buf_cap == 0) { - buf_cap = o + bytes; - growing = true; - } - while(o + bytes > buf_cap){ - buf_cap = buf_cap * 2; - growing = true; - } - if (growing){ - buf = (uint8_t*) realloc(buf, buf_cap); - } - - memcpy(&buf[o], ptr, size * nitems); - buf_size += (size*nitems); - } - return nitems; -} - -BufIOWriter::~BufIOWriter() {} - size_t BufIOReader::operator()(void* ptr, size_t size, size_t nitems) { if (rp >= buf_size) return 0; diff --git a/faiss/impl/io.h b/faiss/impl/io.h index e1498c48c1..bcc97f5ab9 100644 --- a/faiss/impl/io.h +++ b/faiss/impl/io.h @@ -61,16 +61,6 @@ struct VectorIOWriter : IOWriter { size_t operator()(const void* ptr, size_t size, size_t nitems) override; }; -struct BufIOWriter : IOWriter { - uint8_t* buf; - size_t buf_size = 0; - // default it to zero. the capacity is to be initialized when - // the read operation is performed for the first time with a chunk of data. - size_t buf_cap = 0; - size_t operator()(const void* ptr, size_t size, size_t nitems) override; - ~BufIOWriter() override; -}; - struct BufIOReader : IOReader { const uint8_t* buf; size_t rp = 0; From 472dc6fdb835053029143ba8663ddcc33b291b9d Mon Sep 17 00:00:00 2001 From: Thejas-bhat Date: Fri, 1 Dec 2023 11:06:15 +0530 Subject: [PATCH 5/5] code comments --- faiss/impl/io.cpp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/faiss/impl/io.cpp b/faiss/impl/io.cpp index ef5e47af9b..9d29b0b807 100644 --- a/faiss/impl/io.cpp +++ b/faiss/impl/io.cpp @@ -60,12 +60,19 @@ size_t VectorIOReader::operator()(void* ptr, size_t size, size_t nitems) { ***********************************************************************/ size_t BufIOReader::operator()(void* ptr, size_t size, size_t nitems) { + // if the read pointer has passed the buffer size, exit out since we've + // read the complete index if (rp >= buf_size) return 0; + + // check how many "items" of a particular size are to be read from the buffer + // the size of the item depends on the datatype of field being populated. size_t nremain = (buf_size - rp) / size; - if (nremain < nitems) - nitems = nremain; + if (nremain < nitems) // we don't have enough items to be read, in which case + nitems = nremain; // read all the remaining ones if (size * nitems > 0) { + // finally memcpy the data from buffer to the field of the index being + // populated and increment the read pointer. memcpy(ptr, &buf[rp], size * nitems); rp += size * nitems; }