From 752f9616c19437db46d15b2df4e7674e1acb34ec Mon Sep 17 00:00:00 2001 From: Junjie Qi Date: Wed, 27 Mar 2024 10:13:35 -0700 Subject: [PATCH 01/24] Revert "remove broken change" This reverts commit 673f48ad26a8b965ccfb683303e3a21ef9688d27. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 549e4a2793..f1c763154d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -266,7 +266,7 @@ jobs: name: C++ tests command: | export GTEST_OUTPUT="xml:$(realpath .)/test-results/googletest/" - make -C build test + make -C build test --output-on-failure - run: name: Install Python extension command: | From 9b6f3412d3dcd84ffa63c3a08be237fcea0fb690 Mon Sep 17 00:00:00 2001 From: Junjie Qi Date: Mon, 8 Apr 2024 18:26:24 -0700 Subject: [PATCH 02/24] fix osx swig --- .circleci/config.yml | 3 +++ faiss/python/swigfaiss.swig | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f1c763154d..cb34c3501c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -346,6 +346,9 @@ workflows: - build_conda: name: Linux arm64 (conda) exec: linux-arm64-cpu + - build_conda: + name: OSX arm64 nightlies + exec: macosx-arm64-cpu - build_conda: name: Linux x86_64 packages exec: linux-x86_64-cpu diff --git a/faiss/python/swigfaiss.swig b/faiss/python/swigfaiss.swig index 0ea93609e3..33d0174189 100644 --- a/faiss/python/swigfaiss.swig +++ b/faiss/python/swigfaiss.swig @@ -1022,14 +1022,14 @@ PyObject *swig_ptr (PyObject *a) return SWIG_NewPointerObj(data, SWIGTYPE_p_bool, 0); } if(PyArray_TYPE(ao) == NPY_UINT64) { -#if (__SIZEOF_LONG__ == 8) +#if defined(__x86_64) || defined(__x86_64__) || (__WORDSIZE == 64) return SWIG_NewPointerObj(data, SWIGTYPE_p_unsigned_long, 0); #else return SWIG_NewPointerObj(data, SWIGTYPE_p_unsigned_long_long, 0); #endif } if(PyArray_TYPE(ao) == NPY_INT64) { -#if (__SIZEOF_LONG__ == 8) +#if defined(__x86_64) || defined(__x86_64__) || (__WORDSIZE == 64) return SWIG_NewPointerObj(data, SWIGTYPE_p_long, 0); #else return SWIG_NewPointerObj(data, SWIGTYPE_p_long_long, 0); From 0da050e51d3f46ee972faeb26b18e21a44bbcbea Mon Sep 17 00:00:00 2001 From: Junjie Qi Date: Mon, 8 Apr 2024 18:28:04 -0700 Subject: [PATCH 03/24] remove unused change --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index cb34c3501c..9c02f8eaf5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -266,7 +266,7 @@ jobs: name: C++ tests command: | export GTEST_OUTPUT="xml:$(realpath .)/test-results/googletest/" - make -C build test --output-on-failure + make -C build test - run: name: Install Python extension command: | From b1f5c1767ed9afc55e1f3f21e57f6b421876156a Mon Sep 17 00:00:00 2001 From: Junjie Qi Date: Mon, 8 Apr 2024 22:29:01 -0700 Subject: [PATCH 04/24] fix sprintf --- faiss/utils/simdlib_neon.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/faiss/utils/simdlib_neon.h b/faiss/utils/simdlib_neon.h index 656a561217..99569c6e9c 100644 --- a/faiss/utils/simdlib_neon.h +++ b/faiss/utils/simdlib_neon.h @@ -168,7 +168,7 @@ static inline std::string elements_to_string(const char* fmt, const S& simd) { simd.store(bytes); char res[1000], *ptr = res; for (size_t i = 0; i < N; ++i) { - ptr += sprintf(ptr, fmt, bytes[i]); + ptr += snprintf(ptr, fmt, bytes[i]); } // strip last , ptr[-1] = 0; From 74ae852573e52481f9428aede14d48cbe1a9cd0a Mon Sep 17 00:00:00 2001 From: Junjie Qi Date: Mon, 8 Apr 2024 22:35:25 -0700 Subject: [PATCH 05/24] revert --- faiss/utils/simdlib_neon.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/faiss/utils/simdlib_neon.h b/faiss/utils/simdlib_neon.h index 99569c6e9c..656a561217 100644 --- a/faiss/utils/simdlib_neon.h +++ b/faiss/utils/simdlib_neon.h @@ -168,7 +168,7 @@ static inline std::string elements_to_string(const char* fmt, const S& simd) { simd.store(bytes); char res[1000], *ptr = res; for (size_t i = 0; i < N; ++i) { - ptr += snprintf(ptr, fmt, bytes[i]); + ptr += sprintf(ptr, fmt, bytes[i]); } // strip last , ptr[-1] = 0; From 739e5a1db646af686e983d4b8187fb4af3cad3bd Mon Sep 17 00:00:00 2001 From: Junjie Qi Date: Mon, 8 Apr 2024 22:47:13 -0700 Subject: [PATCH 06/24] build_cmake --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9c02f8eaf5..76d27adbda 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -346,8 +346,8 @@ workflows: - build_conda: name: Linux arm64 (conda) exec: linux-arm64-cpu - - build_conda: - name: OSX arm64 nightlies + - build_cmake: + name: OSX arm64 (cmake) exec: macosx-arm64-cpu - build_conda: name: Linux x86_64 packages From 609fc2c61254b30eaf9469565538fc216f5123b7 Mon Sep 17 00:00:00 2001 From: Junjie Qi Date: Mon, 8 Apr 2024 22:59:39 -0700 Subject: [PATCH 07/24] snprintf --- faiss/utils/simdlib_neon.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/faiss/utils/simdlib_neon.h b/faiss/utils/simdlib_neon.h index 656a561217..1f6bbcf5a5 100644 --- a/faiss/utils/simdlib_neon.h +++ b/faiss/utils/simdlib_neon.h @@ -168,7 +168,7 @@ static inline std::string elements_to_string(const char* fmt, const S& simd) { simd.store(bytes); char res[1000], *ptr = res; for (size_t i = 0; i < N; ++i) { - ptr += sprintf(ptr, fmt, bytes[i]); + ptr += snprintf(ptr, sizeof(res) fmt, bytes[i]); } // strip last , ptr[-1] = 0; From f77336d5652dde20113171f76289e51273bb0a24 Mon Sep 17 00:00:00 2001 From: Junjie Qi Date: Mon, 8 Apr 2024 23:01:35 -0700 Subject: [PATCH 08/24] revert --- faiss/python/swigfaiss.swig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/faiss/python/swigfaiss.swig b/faiss/python/swigfaiss.swig index 33d0174189..c5366768b8 100644 --- a/faiss/python/swigfaiss.swig +++ b/faiss/python/swigfaiss.swig @@ -1022,14 +1022,14 @@ PyObject *swig_ptr (PyObject *a) return SWIG_NewPointerObj(data, SWIGTYPE_p_bool, 0); } if(PyArray_TYPE(ao) == NPY_UINT64) { -#if defined(__x86_64) || defined(__x86_64__) || (__WORDSIZE == 64) +#if (__SIZEOF_LONG__ == 8) return SWIG_NewPointerObj(data, SWIGTYPE_p_unsigned_long, 0); #else return SWIG_NewPointerObj(data, SWIGTYPE_p_unsigned_long_long, 0); #endif } if(PyArray_TYPE(ao) == NPY_INT64) { -#if defined(__x86_64) || defined(__x86_64__) || (__WORDSIZE == 64) +#if (__SIZEOF_LONG__ == 8) return SWIG_NewPointerObj(data, SWIGTYPE_p_long, 0); #else return SWIG_NewPointerObj(data, SWIGTYPE_p_long_long, 0); From c3b7b989e4fa6c2880b2a9c144c4dfce0f28f4c4 Mon Sep 17 00:00:00 2001 From: Junjie Qi Date: Mon, 8 Apr 2024 23:03:46 -0700 Subject: [PATCH 09/24] revert --- faiss/utils/simdlib_neon.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/faiss/utils/simdlib_neon.h b/faiss/utils/simdlib_neon.h index 1f6bbcf5a5..656a561217 100644 --- a/faiss/utils/simdlib_neon.h +++ b/faiss/utils/simdlib_neon.h @@ -168,7 +168,7 @@ static inline std::string elements_to_string(const char* fmt, const S& simd) { simd.store(bytes); char res[1000], *ptr = res; for (size_t i = 0; i < N; ++i) { - ptr += snprintf(ptr, sizeof(res) fmt, bytes[i]); + ptr += sprintf(ptr, fmt, bytes[i]); } // strip last , ptr[-1] = 0; From e3244ce28005471a509b24601503abfc92d07d96 Mon Sep 17 00:00:00 2001 From: Junjie Qi Date: Mon, 8 Apr 2024 23:06:42 -0700 Subject: [PATCH 10/24] change to conda --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 76d27adbda..bda432b785 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -346,7 +346,7 @@ workflows: - build_conda: name: Linux arm64 (conda) exec: linux-arm64-cpu - - build_cmake: + - build_conda: name: OSX arm64 (cmake) exec: macosx-arm64-cpu - build_conda: From 38929fc9b448f640c4ab2becad8a1452b192a062 Mon Sep 17 00:00:00 2001 From: Junjie Qi Date: Thu, 11 Apr 2024 12:20:38 -0700 Subject: [PATCH 11/24] fix sprintf --- faiss/impl/io.cpp | 2 +- faiss/utils/simdlib_neon.h | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/faiss/impl/io.cpp b/faiss/impl/io.cpp index 3d3af95036..5f5b2d5ebd 100644 --- a/faiss/impl/io.cpp +++ b/faiss/impl/io.cpp @@ -267,7 +267,7 @@ std::string fourcc_inv_printable(uint32_t x) { str += c; } else { char buf[10]; - sprintf(buf, "\\x%02x", c); + snprintf(buf, sizeof(buf), "\\x%02x", c); str += buf; } } diff --git a/faiss/utils/simdlib_neon.h b/faiss/utils/simdlib_neon.h index 656a561217..16fcbb351b 100644 --- a/faiss/utils/simdlib_neon.h +++ b/faiss/utils/simdlib_neon.h @@ -168,10 +168,18 @@ static inline std::string elements_to_string(const char* fmt, const S& simd) { simd.store(bytes); char res[1000], *ptr = res; for (size_t i = 0; i < N; ++i) { - ptr += sprintf(ptr, fmt, bytes[i]); + int bytesWritten = snprintf(ptr, sizeof(res) - (ptr - res), fmt, bytes[i]); + if (bytesWritten >= 0) { + ptr += bytesWritten; + } else { + // Handle error + break; + } } // strip last , - ptr[-1] = 0; + if (ptr != res) { + ptr[-1] = 0; + } return std::string(res); } From 6fa55b0d6debdddbab938fcfba57ae6945195a78 Mon Sep 17 00:00:00 2001 From: Junjie Qi Date: Thu, 11 Apr 2024 15:13:15 -0700 Subject: [PATCH 12/24] force generate SWIGTYPE_p_*_long --- faiss/python/swigfaiss.swig | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/faiss/python/swigfaiss.swig b/faiss/python/swigfaiss.swig index c5366768b8..149b0edca6 100644 --- a/faiss/python/swigfaiss.swig +++ b/faiss/python/swigfaiss.swig @@ -285,8 +285,8 @@ namespace std { %include %include -%include %include +%include int get_num_gpus(); void gpu_profiler_start(); @@ -1123,10 +1123,14 @@ void * cast_integer_to_void_ptr (int64_t x) { } %} +/* +Force OSX to generate SWIGTYPE_p_long since OSX will generate SWIGTYPE_p_long_long when it meets int64 by default +*/ +%inline %{ +void dummy_function(long *) {} - - - +void dummy_unsigned_function(unsigned long *) {} +%} From e3eb21c6de3dcd2893a20d03078bb37f31685805 Mon Sep 17 00:00:00 2001 From: Junjie Qi Date: Thu, 11 Apr 2024 15:25:16 -0700 Subject: [PATCH 13/24] revert some part --- faiss/impl/io.cpp | 2 +- faiss/utils/simdlib_neon.h | 12 ++---------- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/faiss/impl/io.cpp b/faiss/impl/io.cpp index 5f5b2d5ebd..3d3af95036 100644 --- a/faiss/impl/io.cpp +++ b/faiss/impl/io.cpp @@ -267,7 +267,7 @@ std::string fourcc_inv_printable(uint32_t x) { str += c; } else { char buf[10]; - snprintf(buf, sizeof(buf), "\\x%02x", c); + sprintf(buf, "\\x%02x", c); str += buf; } } diff --git a/faiss/utils/simdlib_neon.h b/faiss/utils/simdlib_neon.h index 16fcbb351b..656a561217 100644 --- a/faiss/utils/simdlib_neon.h +++ b/faiss/utils/simdlib_neon.h @@ -168,18 +168,10 @@ static inline std::string elements_to_string(const char* fmt, const S& simd) { simd.store(bytes); char res[1000], *ptr = res; for (size_t i = 0; i < N; ++i) { - int bytesWritten = snprintf(ptr, sizeof(res) - (ptr - res), fmt, bytes[i]); - if (bytesWritten >= 0) { - ptr += bytesWritten; - } else { - // Handle error - break; - } + ptr += sprintf(ptr, fmt, bytes[i]); } // strip last , - if (ptr != res) { - ptr[-1] = 0; - } + ptr[-1] = 0; return std::string(res); } From 8b4ffc34904e578b800d6203864f2f395c2725b4 Mon Sep 17 00:00:00 2001 From: Junjie Qi Date: Thu, 11 Apr 2024 23:07:43 -0700 Subject: [PATCH 14/24] reconstructure --- faiss/python/swigfaiss.swig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/faiss/python/swigfaiss.swig b/faiss/python/swigfaiss.swig index 149b0edca6..da303fa99c 100644 --- a/faiss/python/swigfaiss.swig +++ b/faiss/python/swigfaiss.swig @@ -285,8 +285,8 @@ namespace std { %include %include -%include %include +%include int get_num_gpus(); void gpu_profiler_start(); From deb576ba12869cacb78f9adffa5895756cae9452 Mon Sep 17 00:00:00 2001 From: Junjie Qi Date: Fri, 12 Apr 2024 15:40:27 -0700 Subject: [PATCH 15/24] force long --- .circleci/config.yml | 70 ++++++++++++++++++------------------- faiss/python/swigfaiss.swig | 9 ++--- 2 files changed, 38 insertions(+), 41 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index bda432b785..3c042e0a92 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -311,41 +311,41 @@ workflows: version: 2 build: jobs: - - format: - name: Format - - build_cmake: - name: Linux x86_64 (cmake) - exec: linux-x86_64-cpu - - build_cmake: - name: Linux x86_64 AVX2 (cmake) - exec: linux-x86_64-cpu - opt_level: "avx2" - - build_cmake: - name: Linux x86_64 AVX512 (cmake) - exec: linux-x86_64-cpu - opt_level: "avx512" - - build_cmake: - name: Linux x86_64 GPU (cmake) - exec: linux-x86_64-gpu - gpu: "ON" - requires: - - Linux x86_64 AVX2 (cmake) - - build_cmake: - name: Linux x86_64 GPU w/ RAFT (cmake) - exec: linux-x86_64-gpu - gpu: "ON" - raft: "ON" - requires: - - Linux x86_64 GPU (cmake) - - build_conda: - name: Linux x86_64 (conda) - exec: linux-x86_64-cpu - - build_conda: - name: Windows x86_64 (conda) - exec: windows-x86_64-cpu - - build_conda: - name: Linux arm64 (conda) - exec: linux-arm64-cpu + # - format: + # name: Format + # - build_cmake: + # name: Linux x86_64 (cmake) + # exec: linux-x86_64-cpu + # - build_cmake: + # name: Linux x86_64 AVX2 (cmake) + # exec: linux-x86_64-cpu + # opt_level: "avx2" + # - build_cmake: + # name: Linux x86_64 AVX512 (cmake) + # exec: linux-x86_64-cpu + # opt_level: "avx512" + # - build_cmake: + # name: Linux x86_64 GPU (cmake) + # exec: linux-x86_64-gpu + # gpu: "ON" + # requires: + # - Linux x86_64 AVX2 (cmake) + # - build_cmake: + # name: Linux x86_64 GPU w/ RAFT (cmake) + # exec: linux-x86_64-gpu + # gpu: "ON" + # raft: "ON" + # requires: + # - Linux x86_64 GPU (cmake) + # - build_conda: + # name: Linux x86_64 (conda) + # exec: linux-x86_64-cpu + # - build_conda: + # name: Windows x86_64 (conda) + # exec: windows-x86_64-cpu + # - build_conda: + # name: Linux arm64 (conda) + # exec: linux-arm64-cpu - build_conda: name: OSX arm64 (cmake) exec: macosx-arm64-cpu diff --git a/faiss/python/swigfaiss.swig b/faiss/python/swigfaiss.swig index da303fa99c..cdf8760547 100644 --- a/faiss/python/swigfaiss.swig +++ b/faiss/python/swigfaiss.swig @@ -1121,16 +1121,13 @@ int * cast_integer_to_int_ptr (int64_t x) { void * cast_integer_to_void_ptr (int64_t x) { return (void*)x; } - %} + /* Force OSX to generate SWIGTYPE_p_long since OSX will generate SWIGTYPE_p_long_long when it meets int64 by default */ -%inline %{ -void dummy_function(long *) {} - -void dummy_unsigned_function(unsigned long *) {} -%} +%types(unsigned long*); +%types(long*); From 47244caf426375dd8cbb9abfd7da5dae94b55c82 Mon Sep 17 00:00:00 2001 From: Junjie Qi Date: Fri, 12 Apr 2024 18:09:15 -0700 Subject: [PATCH 16/24] remove unused flag --- faiss/python/CMakeLists.txt | 5 ----- 1 file changed, 5 deletions(-) diff --git a/faiss/python/CMakeLists.txt b/faiss/python/CMakeLists.txt index 8bca710f5f..ae14b5299f 100644 --- a/faiss/python/CMakeLists.txt +++ b/faiss/python/CMakeLists.txt @@ -24,11 +24,6 @@ macro(configure_swigfaiss source) CPLUSPLUS ON USE_TARGET_INCLUDE_DIRECTORIES TRUE ) - if(NOT ${CMAKE_SYSTEM_NAME} MATCHES "Darwin" AND NOT WIN32) - set_source_files_properties(${source} PROPERTIES - SWIG_FLAGS -DSWIGWORDSIZE64 - ) - endif() if(WIN32) set_source_files_properties(${source} PROPERTIES SWIG_FLAGS -DSWIGWIN From 736b4eebdb191a45c95bd10396b08b14717b9089 Mon Sep 17 00:00:00 2001 From: Junjie Qi Date: Wed, 24 Apr 2024 12:44:28 -0700 Subject: [PATCH 17/24] convert int64 to long_long for osx --- faiss/python/swigfaiss.swig | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/faiss/python/swigfaiss.swig b/faiss/python/swigfaiss.swig index cdf8760547..d8bce9968d 100644 --- a/faiss/python/swigfaiss.swig +++ b/faiss/python/swigfaiss.swig @@ -1022,14 +1022,17 @@ PyObject *swig_ptr (PyObject *a) return SWIG_NewPointerObj(data, SWIGTYPE_p_bool, 0); } if(PyArray_TYPE(ao) == NPY_UINT64) { -#if (__SIZEOF_LONG__ == 8) + // Convert npy64 either long or long long and it depends on how compiler define int64_t. + // In the 64bit machine, typically the int64_t should be long but it is not hold for Apple osx. + // In this case, we want to convert npy64 to long_Long in osx +#if (__SIZEOF_LONG__ == 8 && !__APPLE__) return SWIG_NewPointerObj(data, SWIGTYPE_p_unsigned_long, 0); #else return SWIG_NewPointerObj(data, SWIGTYPE_p_unsigned_long_long, 0); #endif } if(PyArray_TYPE(ao) == NPY_INT64) { -#if (__SIZEOF_LONG__ == 8) +#if (__SIZEOF_LONG__ == 8 && !__APPLE__) return SWIG_NewPointerObj(data, SWIGTYPE_p_long, 0); #else return SWIG_NewPointerObj(data, SWIGTYPE_p_long_long, 0); From a26c9c2deed0607439155a86ab78f76f2e6c68b3 Mon Sep 17 00:00:00 2001 From: Junjie Qi Date: Wed, 24 Apr 2024 12:54:46 -0700 Subject: [PATCH 18/24] revert the comment --- .circleci/config.yml | 71 +++++++++++++++++++++----------------------- 1 file changed, 34 insertions(+), 37 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 80cd8f8655..7e8bd8170a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -316,44 +316,41 @@ workflows: version: 2 build: jobs: - # - format: - # name: Format - # - build_cmake: - # name: Linux x86_64 (cmake) - # exec: linux-x86_64-cpu - # - build_cmake: - # name: Linux x86_64 AVX2 (cmake) - # exec: linux-x86_64-cpu - # opt_level: "avx2" - # - build_cmake: - # name: Linux x86_64 AVX512 (cmake) - # exec: linux-x86_64-cpu - # opt_level: "avx512" - # - build_cmake: - # name: Linux x86_64 GPU (cmake) - # exec: linux-x86_64-gpu - # gpu: "ON" - # requires: - # - Linux x86_64 AVX2 (cmake) - # - build_cmake: - # name: Linux x86_64 GPU w/ RAFT (cmake) - # exec: linux-x86_64-gpu - # gpu: "ON" - # raft: "ON" - # requires: - # - Linux x86_64 GPU (cmake) - # - build_conda: - # name: Linux x86_64 (conda) - # exec: linux-x86_64-cpu - # - build_conda: - # name: Windows x86_64 (conda) - # exec: windows-x86_64-cpu - # - build_conda: - # name: Linux arm64 (conda) - # exec: linux-arm64-cpu + - format: + name: Format + - build_cmake: + name: Linux x86_64 (cmake) + exec: linux-x86_64-cpu + - build_cmake: + name: Linux x86_64 AVX2 (cmake) + exec: linux-x86_64-cpu + opt_level: "avx2" + - build_cmake: + name: Linux x86_64 AVX512 (cmake) + exec: linux-x86_64-cpu + opt_level: "avx512" + - build_cmake: + name: Linux x86_64 GPU (cmake) + exec: linux-x86_64-gpu + gpu: "ON" + requires: + - Linux x86_64 AVX2 (cmake) + - build_cmake: + name: Linux x86_64 GPU w/ RAFT (cmake) + exec: linux-x86_64-gpu + gpu: "ON" + raft: "ON" + requires: + - Linux x86_64 GPU (cmake) - build_conda: - name: OSX arm64 (cmake) - exec: macosx-arm64-cpu + name: Linux x86_64 (conda) + exec: linux-x86_64-cpu + - build_conda: + name: Windows x86_64 (conda) + exec: windows-x86_64-cpu + - build_conda: + name: Linux arm64 (conda) + exec: linux-arm64-cpu - build_conda: name: Linux x86_64 packages exec: linux-x86_64-cpu From f5e4649d7fb8c0f1c7ed9e0072d1d21c8361c20a Mon Sep 17 00:00:00 2001 From: Junjie Qi Date: Wed, 24 Apr 2024 13:06:09 -0700 Subject: [PATCH 19/24] use defined --- .circleci/config.yml | 3 +++ faiss/python/swigfaiss.swig | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7e8bd8170a..e5fdb70fde 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -351,6 +351,9 @@ workflows: - build_conda: name: Linux arm64 (conda) exec: linux-arm64-cpu + - build_conda: + name: OSX arm64 (cmake) + exec: macosx-arm64-cpu - build_conda: name: Linux x86_64 packages exec: linux-x86_64-cpu diff --git a/faiss/python/swigfaiss.swig b/faiss/python/swigfaiss.swig index d8bce9968d..c5c66508fd 100644 --- a/faiss/python/swigfaiss.swig +++ b/faiss/python/swigfaiss.swig @@ -1025,14 +1025,14 @@ PyObject *swig_ptr (PyObject *a) // Convert npy64 either long or long long and it depends on how compiler define int64_t. // In the 64bit machine, typically the int64_t should be long but it is not hold for Apple osx. // In this case, we want to convert npy64 to long_Long in osx -#if (__SIZEOF_LONG__ == 8 && !__APPLE__) +#if __SIZEOF_LONG__ == 8 && !defined(__APPLE__) return SWIG_NewPointerObj(data, SWIGTYPE_p_unsigned_long, 0); #else return SWIG_NewPointerObj(data, SWIGTYPE_p_unsigned_long_long, 0); #endif } if(PyArray_TYPE(ao) == NPY_INT64) { -#if (__SIZEOF_LONG__ == 8 && !__APPLE__) +#if __SIZEOF_LONG__ == 8 && !defined(__APPLE__) return SWIG_NewPointerObj(data, SWIGTYPE_p_long, 0); #else return SWIG_NewPointerObj(data, SWIGTYPE_p_long_long, 0); From 7c5d14f622a16993ceeb4b834063fba65d22aa0d Mon Sep 17 00:00:00 2001 From: Junjie Qi Date: Wed, 24 Apr 2024 13:14:21 -0700 Subject: [PATCH 20/24] add __MACH__ --- faiss/python/swigfaiss.swig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/faiss/python/swigfaiss.swig b/faiss/python/swigfaiss.swig index c5c66508fd..4905ac98cd 100644 --- a/faiss/python/swigfaiss.swig +++ b/faiss/python/swigfaiss.swig @@ -1025,14 +1025,14 @@ PyObject *swig_ptr (PyObject *a) // Convert npy64 either long or long long and it depends on how compiler define int64_t. // In the 64bit machine, typically the int64_t should be long but it is not hold for Apple osx. // In this case, we want to convert npy64 to long_Long in osx -#if __SIZEOF_LONG__ == 8 && !defined(__APPLE__) +#if __SIZEOF_LONG__ == 8 && !(defined(__APPLE__) && defined(__MACH__)) return SWIG_NewPointerObj(data, SWIGTYPE_p_unsigned_long, 0); #else return SWIG_NewPointerObj(data, SWIGTYPE_p_unsigned_long_long, 0); #endif } if(PyArray_TYPE(ao) == NPY_INT64) { -#if __SIZEOF_LONG__ == 8 && !defined(__APPLE__) +#if __SIZEOF_LONG__ == 8 && !(defined(__APPLE__) && defined(__MACH__)) return SWIG_NewPointerObj(data, SWIGTYPE_p_long, 0); #else return SWIG_NewPointerObj(data, SWIGTYPE_p_long_long, 0); From 32dd565d1c37a5b5a1cc3142071ee7bc3f0cc1f6 Mon Sep 17 00:00:00 2001 From: Junjie Qi Date: Wed, 24 Apr 2024 17:23:49 -0700 Subject: [PATCH 21/24] __APPLE__ --- faiss/python/swigfaiss.swig | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/faiss/python/swigfaiss.swig b/faiss/python/swigfaiss.swig index 4905ac98cd..86dc3ef88b 100644 --- a/faiss/python/swigfaiss.swig +++ b/faiss/python/swigfaiss.swig @@ -1025,14 +1025,14 @@ PyObject *swig_ptr (PyObject *a) // Convert npy64 either long or long long and it depends on how compiler define int64_t. // In the 64bit machine, typically the int64_t should be long but it is not hold for Apple osx. // In this case, we want to convert npy64 to long_Long in osx -#if __SIZEOF_LONG__ == 8 && !(defined(__APPLE__) && defined(__MACH__)) +#ifndef __APPLE__ return SWIG_NewPointerObj(data, SWIGTYPE_p_unsigned_long, 0); #else return SWIG_NewPointerObj(data, SWIGTYPE_p_unsigned_long_long, 0); #endif } if(PyArray_TYPE(ao) == NPY_INT64) { -#if __SIZEOF_LONG__ == 8 && !(defined(__APPLE__) && defined(__MACH__)) +#ifndef __APPLE__ return SWIG_NewPointerObj(data, SWIGTYPE_p_long, 0); #else return SWIG_NewPointerObj(data, SWIGTYPE_p_long_long, 0); @@ -1057,11 +1057,8 @@ struct PythonInterruptCallback: faiss::InterruptCallback { } }; - - %} - %init %{ /* needed, else crash at runtime */ import_array(); @@ -1126,14 +1123,6 @@ void * cast_integer_to_void_ptr (int64_t x) { } %} -/* -Force OSX to generate SWIGTYPE_p_long since OSX will generate SWIGTYPE_p_long_long when it meets int64 by default -*/ -%types(unsigned long*); -%types(long*); - - - %inline %{ void wait() { // in gdb, use return to get out of this function From 6d45822f9f86fd2125393a8387fee7a9ec281236 Mon Sep 17 00:00:00 2001 From: Junjie Qi Date: Thu, 25 Apr 2024 00:00:02 -0700 Subject: [PATCH 22/24] add DSWIGWORDSIZE64 --- faiss/python/CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/faiss/python/CMakeLists.txt b/faiss/python/CMakeLists.txt index f9aaeb5998..dee8c7762e 100644 --- a/faiss/python/CMakeLists.txt +++ b/faiss/python/CMakeLists.txt @@ -24,6 +24,11 @@ macro(configure_swigfaiss source) CPLUSPLUS ON USE_TARGET_INCLUDE_DIRECTORIES TRUE ) + if(NOT ${CMAKE_SYSTEM_NAME} MATCHES "Darwin" AND NOT WIN32) + set_source_files_properties(${source} PROPERTIES + SWIG_FLAGS -DSWIGWORDSIZE64 + ) + endif() if(WIN32) set_source_files_properties(${source} PROPERTIES SWIG_FLAGS -DSWIGWIN From aab10badc4fe685c3c7c60217160f33e04fcd189 Mon Sep 17 00:00:00 2001 From: Junjie Qi Date: Thu, 25 Apr 2024 00:30:38 -0700 Subject: [PATCH 23/24] add size back --- .circleci/config.yml | 3 +++ faiss/python/swigfaiss.swig | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7e8bd8170a..3ebff0e829 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -351,6 +351,9 @@ workflows: - build_conda: name: Linux arm64 (conda) exec: linux-arm64-cpu + - build_conda: + name: OSX arm64 (conda) + exec: macosx-arm64-cpu - build_conda: name: Linux x86_64 packages exec: linux-x86_64-cpu diff --git a/faiss/python/swigfaiss.swig b/faiss/python/swigfaiss.swig index 86dc3ef88b..5c9a7b3fa7 100644 --- a/faiss/python/swigfaiss.swig +++ b/faiss/python/swigfaiss.swig @@ -1025,14 +1025,14 @@ PyObject *swig_ptr (PyObject *a) // Convert npy64 either long or long long and it depends on how compiler define int64_t. // In the 64bit machine, typically the int64_t should be long but it is not hold for Apple osx. // In this case, we want to convert npy64 to long_Long in osx -#ifndef __APPLE__ +#if __SIZEOF_LONG__ == 8 && !defined(__APPLE__) return SWIG_NewPointerObj(data, SWIGTYPE_p_unsigned_long, 0); #else return SWIG_NewPointerObj(data, SWIGTYPE_p_unsigned_long_long, 0); #endif } if(PyArray_TYPE(ao) == NPY_INT64) { -#ifndef __APPLE__ +#if __SIZEOF_LONG__ == 8 && !defined(__APPLE__) return SWIG_NewPointerObj(data, SWIGTYPE_p_long, 0); #else return SWIG_NewPointerObj(data, SWIGTYPE_p_long_long, 0); From bba54ac467d5f206048ee84fba72a925266cbc93 Mon Sep 17 00:00:00 2001 From: Junjie Qi Date: Thu, 25 Apr 2024 00:54:30 -0700 Subject: [PATCH 24/24] remove osx for final check in --- .circleci/config.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3ebff0e829..7e8bd8170a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -351,9 +351,6 @@ workflows: - build_conda: name: Linux arm64 (conda) exec: linux-arm64-cpu - - build_conda: - name: OSX arm64 (conda) - exec: macosx-arm64-cpu - build_conda: name: Linux x86_64 packages exec: linux-x86_64-cpu