From e008f02ffe4ffcdfbe821d05862accd5e4d978df Mon Sep 17 00:00:00 2001 From: mengdilin Date: Mon, 7 Oct 2024 06:35:38 -0700 Subject: [PATCH 01/16] disable benchmarks --- .github/actions/build_cmake/action.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/actions/build_cmake/action.yml b/.github/actions/build_cmake/action.yml index 3fad247b6d..fcfa1cfa37 100644 --- a/.github/actions/build_cmake/action.yml +++ b/.github/actions/build_cmake/action.yml @@ -143,11 +143,11 @@ runs: run: | export GTEST_OUTPUT="xml:$(realpath .)/test-results/googletest/" make -C build test - - name: C++ perf benchmarks - shell: bash - if: inputs.rocm == 'OFF' - run: | - find ./build/perf_tests/ -executable -type f -name "bench*" -exec '{}' -v \; + # - name: C++ perf benchmarks + # shell: bash + # if: inputs.rocm == 'OFF' + # run: | + # find ./build/perf_tests/ -executable -type f -name "bench*" -exec '{}' -v \; - name: Install Python extension shell: bash working-directory: build/faiss/python From 773b2b817a4580160a8b5ccd11262950f3e7290f Mon Sep 17 00:00:00 2001 From: mengdilin Date: Mon, 7 Oct 2024 09:49:54 -0700 Subject: [PATCH 02/16] generic build --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a17f0838b1..4d2ee24584 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -122,8 +122,8 @@ jobs: uses: actions/checkout@v4 - name: Build and Test (cmake) uses: ./.github/actions/build_cmake - with: - opt_level: sve + # with: + # opt_level: sve linux-x86_64-conda: name: Linux x86_64 (conda) needs: linux-x86_64-cmake From dc31f0d0f411ee8e247f0c0722a0e8ca2816a4e8 Mon Sep 17 00:00:00 2001 From: mengdilin Date: Mon, 7 Oct 2024 09:51:22 -0700 Subject: [PATCH 03/16] remove perf_tests from compilation --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c4fea46a3d..6d561a3357 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -110,7 +110,7 @@ add_subdirectory(tutorial/cpp) include(CTest) if(BUILD_TESTING) add_subdirectory(tests) - add_subdirectory(perf_tests) + # add_subdirectory(perf_tests) if(FAISS_ENABLE_GPU) if(FAISS_ENABLE_ROCM) add_subdirectory(faiss/gpu-rocm/test) From fb06c7d70cb665abd3b99c116a63bfb4dc9aaf64 Mon Sep 17 00:00:00 2001 From: mengdilin Date: Mon, 7 Oct 2024 11:45:42 -0700 Subject: [PATCH 04/16] use old openblas --- .github/actions/build_cmake/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/build_cmake/action.yml b/.github/actions/build_cmake/action.yml index fcfa1cfa37..49257d8030 100644 --- a/.github/actions/build_cmake/action.yml +++ b/.github/actions/build_cmake/action.yml @@ -37,11 +37,11 @@ runs: # install base packages for ARM64 if [ "${{ runner.arch }}" = "ARM64" ]; then # installing libmamba-solver is necessary for openblas=*=*openmp* - conda install --solver=classic conda-forge::conda-libmamba-solver conda-forge::libmamba conda-forge::libmambapy conda-forge::libarchive + # conda install --solver=classic conda-forge::conda-libmamba-solver conda-forge::libmamba conda-forge::libmambapy conda-forge::libarchive # installing openblas that was compiled with USE_OPENMP=1. # Context: https://github.com/facebookresearch/faiss/wiki/Troubleshooting#surprising-faiss-openmp-and-openblas-interaction - conda install -y -q -c conda-forge openblas=*=*openmp* gxx_linux-aarch64 sysroot_linux-aarch64 + conda install -y -q -c conda-forge openblas gxx_linux-aarch64 sysroot_linux-aarch64 fi # install base packages for X86_64 From 4985840eeac984a93be3d8a022c1af05d68c2136 Mon Sep 17 00:00:00 2001 From: mengdilin Date: Mon, 7 Oct 2024 12:17:22 -0700 Subject: [PATCH 05/16] re-enable perf --- .github/actions/build_cmake/action.yml | 10 +++++----- CMakeLists.txt | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/actions/build_cmake/action.yml b/.github/actions/build_cmake/action.yml index 49257d8030..72ca38bdce 100644 --- a/.github/actions/build_cmake/action.yml +++ b/.github/actions/build_cmake/action.yml @@ -143,11 +143,11 @@ runs: run: | export GTEST_OUTPUT="xml:$(realpath .)/test-results/googletest/" make -C build test - # - name: C++ perf benchmarks - # shell: bash - # if: inputs.rocm == 'OFF' - # run: | - # find ./build/perf_tests/ -executable -type f -name "bench*" -exec '{}' -v \; + - name: C++ perf benchmarks + shell: bash + if: inputs.rocm == 'OFF' + run: | + find ./build/perf_tests/ -executable -type f -name "bench*" -exec '{}' -v \; - name: Install Python extension shell: bash working-directory: build/faiss/python diff --git a/CMakeLists.txt b/CMakeLists.txt index 6d561a3357..c4fea46a3d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -110,7 +110,7 @@ add_subdirectory(tutorial/cpp) include(CTest) if(BUILD_TESTING) add_subdirectory(tests) - # add_subdirectory(perf_tests) + add_subdirectory(perf_tests) if(FAISS_ENABLE_GPU) if(FAISS_ENABLE_ROCM) add_subdirectory(faiss/gpu-rocm/test) From 60959bc3c82ed976974cfe8080d6b75153964a4b Mon Sep 17 00:00:00 2001 From: mengdilin Date: Mon, 7 Oct 2024 12:22:04 -0700 Subject: [PATCH 06/16] re-enable sve --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4d2ee24584..a17f0838b1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -122,8 +122,8 @@ jobs: uses: actions/checkout@v4 - name: Build and Test (cmake) uses: ./.github/actions/build_cmake - # with: - # opt_level: sve + with: + opt_level: sve linux-x86_64-conda: name: Linux x86_64 (conda) needs: linux-x86_64-cmake From 9a37a5d9978077aa15193a5527ce057952cbe6e9 Mon Sep 17 00:00:00 2001 From: mengdilin Date: Mon, 7 Oct 2024 12:23:42 -0700 Subject: [PATCH 07/16] libmamba solver version --- .github/actions/build_cmake/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/build_cmake/action.yml b/.github/actions/build_cmake/action.yml index 72ca38bdce..fe6e8dfc75 100644 --- a/.github/actions/build_cmake/action.yml +++ b/.github/actions/build_cmake/action.yml @@ -37,7 +37,7 @@ runs: # install base packages for ARM64 if [ "${{ runner.arch }}" = "ARM64" ]; then # installing libmamba-solver is necessary for openblas=*=*openmp* - # conda install --solver=classic conda-forge::conda-libmamba-solver conda-forge::libmamba conda-forge::libmambapy conda-forge::libarchive + # conda install --solver=classic conda-forge::conda-libmamba-solver=24.7.0 conda-forge::libmamba conda-forge::libmambapy conda-forge::libarchive # installing openblas that was compiled with USE_OPENMP=1. # Context: https://github.com/facebookresearch/faiss/wiki/Troubleshooting#surprising-faiss-openmp-and-openblas-interaction From 26fb066ed6335b550b0d241bbbfa4988e5497f17 Mon Sep 17 00:00:00 2001 From: mengdilin Date: Mon, 7 Oct 2024 12:25:03 -0700 Subject: [PATCH 08/16] pin libmamba-solver --- .github/actions/build_cmake/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/build_cmake/action.yml b/.github/actions/build_cmake/action.yml index fe6e8dfc75..75e1127121 100644 --- a/.github/actions/build_cmake/action.yml +++ b/.github/actions/build_cmake/action.yml @@ -37,7 +37,7 @@ runs: # install base packages for ARM64 if [ "${{ runner.arch }}" = "ARM64" ]; then # installing libmamba-solver is necessary for openblas=*=*openmp* - # conda install --solver=classic conda-forge::conda-libmamba-solver=24.7.0 conda-forge::libmamba conda-forge::libmambapy conda-forge::libarchive + conda install --solver=classic conda-forge::conda-libmamba-solver=24.7.0 conda-forge::libmamba conda-forge::libmambapy conda-forge::libarchive # installing openblas that was compiled with USE_OPENMP=1. # Context: https://github.com/facebookresearch/faiss/wiki/Troubleshooting#surprising-faiss-openmp-and-openblas-interaction From 0e8f2c8267aee6b3ae822ef9768af01a845fbbb7 Mon Sep 17 00:00:00 2001 From: mengdilin Date: Mon, 7 Oct 2024 12:25:26 -0700 Subject: [PATCH 09/16] re-enable openmp openblas --- .github/actions/build_cmake/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/build_cmake/action.yml b/.github/actions/build_cmake/action.yml index 75e1127121..b091dcc792 100644 --- a/.github/actions/build_cmake/action.yml +++ b/.github/actions/build_cmake/action.yml @@ -41,7 +41,7 @@ runs: # installing openblas that was compiled with USE_OPENMP=1. # Context: https://github.com/facebookresearch/faiss/wiki/Troubleshooting#surprising-faiss-openmp-and-openblas-interaction - conda install -y -q -c conda-forge openblas gxx_linux-aarch64 sysroot_linux-aarch64 + conda install -y -q -c conda-forge openblas=*=*openmp* gxx_linux-aarch64 sysroot_linux-aarch64 fi # install base packages for X86_64 From 616dd902e50d520eeec4a14ac7e4641d34c09050 Mon Sep 17 00:00:00 2001 From: mengdilin Date: Mon, 7 Oct 2024 12:26:16 -0700 Subject: [PATCH 10/16] debug: remove dep --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a17f0838b1..5707d453aa 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -115,7 +115,7 @@ jobs: rocm: ON linux-arm64-SVE-cmake: name: Linux arm64 SVE (cmake) - needs: linux-x86_64-cmake + # needs: linux-x86_64-cmake runs-on: faiss-aws-r8g.large steps: - name: Checkout From 1f602b61c5e830f94f8f531db038c35b461918fe Mon Sep 17 00:00:00 2001 From: mengdilin Date: Mon, 7 Oct 2024 12:45:39 -0700 Subject: [PATCH 11/16] force openblas --- .github/actions/build_cmake/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/build_cmake/action.yml b/.github/actions/build_cmake/action.yml index b091dcc792..f76f871961 100644 --- a/.github/actions/build_cmake/action.yml +++ b/.github/actions/build_cmake/action.yml @@ -134,7 +134,7 @@ runs: -DFAISS_ENABLE_C_API=ON \ -DPYTHON_EXECUTABLE=$CONDA/bin/python \ -DCMAKE_BUILD_TYPE=Release \ - -DBLA_VENDOR=${{ runner.arch == 'X64' && 'Intel10_64_dyn' || '' }} \ + -DBLA_VENDOR=${{ runner.arch == 'X64' && 'Intel10_64_dyn' || runner.arch == 'ARM64' && 'OpenBLAS' }} \ -DCMAKE_CUDA_FLAGS=${{ runner.arch == 'X64' && '"-gencode arch=compute_75,code=sm_75"' || '' }} \ . make -k -C build -j$(nproc) From 5dbd2d47ed5ee282015846d9077f3648ac771ab4 Mon Sep 17 00:00:00 2001 From: mengdilin Date: Mon, 7 Oct 2024 13:41:01 -0700 Subject: [PATCH 12/16] try again --- .github/actions/build_cmake/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/build_cmake/action.yml b/.github/actions/build_cmake/action.yml index f76f871961..f02ce0367a 100644 --- a/.github/actions/build_cmake/action.yml +++ b/.github/actions/build_cmake/action.yml @@ -37,7 +37,7 @@ runs: # install base packages for ARM64 if [ "${{ runner.arch }}" = "ARM64" ]; then # installing libmamba-solver is necessary for openblas=*=*openmp* - conda install --solver=classic conda-forge::conda-libmamba-solver=24.7.0 conda-forge::libmamba conda-forge::libmambapy conda-forge::libarchive + # conda install --solver=classic conda-forge::conda-libmamba-solver=24.7.0 conda-forge::libmamba conda-forge::libmambapy conda-forge::libarchive # installing openblas that was compiled with USE_OPENMP=1. # Context: https://github.com/facebookresearch/faiss/wiki/Troubleshooting#surprising-faiss-openmp-and-openblas-interaction From 9b421bdb1945d7272336ab53bf1e5ade30188ffc Mon Sep 17 00:00:00 2001 From: mengdilin Date: Mon, 7 Oct 2024 14:06:06 -0700 Subject: [PATCH 13/16] env var --- .github/actions/build_cmake/action.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/actions/build_cmake/action.yml b/.github/actions/build_cmake/action.yml index f02ce0367a..5d3a4b9bde 100644 --- a/.github/actions/build_cmake/action.yml +++ b/.github/actions/build_cmake/action.yml @@ -36,12 +36,11 @@ runs: # install base packages for ARM64 if [ "${{ runner.arch }}" = "ARM64" ]; then - # installing libmamba-solver is necessary for openblas=*=*openmp* - # conda install --solver=classic conda-forge::conda-libmamba-solver=24.7.0 conda-forge::libmamba conda-forge::libmambapy conda-forge::libarchive - - # installing openblas that was compiled with USE_OPENMP=1. + # TODO: We should install openblas=*=*openmp* that was compiled with USE_OPENMP=1 # Context: https://github.com/facebookresearch/faiss/wiki/Troubleshooting#surprising-faiss-openmp-and-openblas-interaction - conda install -y -q -c conda-forge openblas=*=*openmp* gxx_linux-aarch64 sysroot_linux-aarch64 + # However that package causes 7-8x slow-down on CI. Figure out why. + conda install -y -q -c conda-forge openblas gxx_linux-aarch64 sysroot_linux-aarch64 + echo "OPENBLAS_NUM_THREADS=4" >> "$GITHUB_ENV" fi # install base packages for X86_64 @@ -134,7 +133,7 @@ runs: -DFAISS_ENABLE_C_API=ON \ -DPYTHON_EXECUTABLE=$CONDA/bin/python \ -DCMAKE_BUILD_TYPE=Release \ - -DBLA_VENDOR=${{ runner.arch == 'X64' && 'Intel10_64_dyn' || runner.arch == 'ARM64' && 'OpenBLAS' }} \ + -DBLA_VENDOR=${{ runner.arch == 'X64' && 'Intel10_64_dyn' || '' }} \ -DCMAKE_CUDA_FLAGS=${{ runner.arch == 'X64' && '"-gencode arch=compute_75,code=sm_75"' || '' }} \ . make -k -C build -j$(nproc) From b97daa435692dfe0056672fb7795cd9ae7add629 Mon Sep 17 00:00:00 2001 From: mengdilin Date: Mon, 7 Oct 2024 14:06:48 -0700 Subject: [PATCH 14/16] single thread --- .github/actions/build_cmake/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/build_cmake/action.yml b/.github/actions/build_cmake/action.yml index 5d3a4b9bde..44084fa308 100644 --- a/.github/actions/build_cmake/action.yml +++ b/.github/actions/build_cmake/action.yml @@ -40,7 +40,7 @@ runs: # Context: https://github.com/facebookresearch/faiss/wiki/Troubleshooting#surprising-faiss-openmp-and-openblas-interaction # However that package causes 7-8x slow-down on CI. Figure out why. conda install -y -q -c conda-forge openblas gxx_linux-aarch64 sysroot_linux-aarch64 - echo "OPENBLAS_NUM_THREADS=4" >> "$GITHUB_ENV" + echo "OPENBLAS_NUM_THREADS=1" >> "$GITHUB_ENV" fi # install base packages for X86_64 From e72c375058fd25e6ad2e22cfc33ca77f6d33ea12 Mon Sep 17 00:00:00 2001 From: mengdilin Date: Tue, 8 Oct 2024 10:43:11 -0700 Subject: [PATCH 15/16] env var --- .github/actions/build_cmake/action.yml | 4 ---- .github/workflows/build.yml | 3 +++ 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/actions/build_cmake/action.yml b/.github/actions/build_cmake/action.yml index 44084fa308..5facd0f0b8 100644 --- a/.github/actions/build_cmake/action.yml +++ b/.github/actions/build_cmake/action.yml @@ -36,11 +36,7 @@ runs: # install base packages for ARM64 if [ "${{ runner.arch }}" = "ARM64" ]; then - # TODO: We should install openblas=*=*openmp* that was compiled with USE_OPENMP=1 - # Context: https://github.com/facebookresearch/faiss/wiki/Troubleshooting#surprising-faiss-openmp-and-openblas-interaction - # However that package causes 7-8x slow-down on CI. Figure out why. conda install -y -q -c conda-forge openblas gxx_linux-aarch64 sysroot_linux-aarch64 - echo "OPENBLAS_NUM_THREADS=1" >> "$GITHUB_ENV" fi # install base packages for X86_64 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5707d453aa..6a2ebef037 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -124,6 +124,9 @@ jobs: uses: ./.github/actions/build_cmake with: opt_level: sve + env: + # Context: https://github.com/facebookresearch/faiss/wiki/Troubleshooting#surprising-faiss-openmp-and-openblas-interaction + OPENBLAS_NUM_THREADS: '1' linux-x86_64-conda: name: Linux x86_64 (conda) needs: linux-x86_64-cmake From adbbaea81f81c6295e3ab387d4fbd28c2ac6a661 Mon Sep 17 00:00:00 2001 From: mengdilin Date: Tue, 8 Oct 2024 12:16:31 -0700 Subject: [PATCH 16/16] fix --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6a2ebef037..b28ba95d1f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -115,7 +115,7 @@ jobs: rocm: ON linux-arm64-SVE-cmake: name: Linux arm64 SVE (cmake) - # needs: linux-x86_64-cmake + needs: linux-x86_64-cmake runs-on: faiss-aws-r8g.large steps: - name: Checkout