From cb3cf152eab864e4ac4aaec945516753ef7e5bf7 Mon Sep 17 00:00:00 2001 From: Junjie Qi Date: Thu, 5 Dec 2024 13:03:59 -0800 Subject: [PATCH 01/20] separare the github build into two conditions --- .github/workflows/build-pull-request.yml | 165 +++++++++++++ .github/workflows/build-release.yml | 148 +++++++++++ .github/workflows/build.yml | 302 +---------------------- 3 files changed, 318 insertions(+), 297 deletions(-) create mode 100644 .github/workflows/build-pull-request.yml create mode 100644 .github/workflows/build-release.yml diff --git a/.github/workflows/build-pull-request.yml b/.github/workflows/build-pull-request.yml new file mode 100644 index 0000000000..c8372fa34c --- /dev/null +++ b/.github/workflows/build-pull-request.yml @@ -0,0 +1,165 @@ +name: Build +on: + workflow_dispatch: + pull_request: + branches: + - main + push: + tags: + - 'v*' +env: + OMP_NUM_THREADS: '10' + MKL_THREADING_LAYER: GNU +jobs: + format: + name: Format + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Install clang-format + run: | + sudo apt-get update -y + sudo apt-get install -y wget + sudo apt install -y lsb-release wget software-properties-common gnupg + wget https://apt.llvm.org/llvm.sh + chmod u+x llvm.sh + sudo ./llvm.sh 18 + sudo apt-get install -y git-core clang-format-18 + - name: Verify clang-format + run: | + git ls-files | grep -E '\.(cpp|h|cu|cuh)$' | xargs clang-format-18 -i + if git diff --quiet; then + echo "Formatting OK!" + else + echo "Formatting not OK!" + echo "------------------" + git --no-pager diff --color + exit 1 + fi + linux-x86_64-cmake: + name: Linux x86_64 (cmake) + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Build and Test (cmake) + uses: ./.github/actions/build_cmake + linux-x86_64-AVX2-cmake: + name: Linux x86_64 AVX2 (cmake) + needs: linux-x86_64-cmake + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Build and Test (cmake) + uses: ./.github/actions/build_cmake + with: + opt_level: avx2 + linux-x86_64-AVX512-cmake: + name: Linux x86_64 AVX512 (cmake) + needs: linux-x86_64-cmake + runs-on: faiss-aws-m7i.large + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Build and Test (cmake) + uses: ./.github/actions/build_cmake + with: + opt_level: avx512 + linux-x86_64-GPU-cmake: + name: Linux x86_64 GPU (cmake) + needs: linux-x86_64-cmake + runs-on: 4-core-ubuntu-gpu-t4 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Build and Test (cmake) + uses: ./.github/actions/build_cmake + with: + gpu: ON + linux-x86_64-GPU-w-CUVS-cmake: + name: Linux x86_64 GPU w/ cuVS (cmake) + needs: linux-x86_64-cmake + runs-on: 4-core-ubuntu-gpu-t4 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Build and Test (cmake) + uses: ./.github/actions/build_cmake + with: + gpu: ON + cuvs: ON + linux-x86_64-GPU-w-ROCm-cmake: + name: Linux x86_64 GPU w/ ROCm (cmake) + needs: linux-x86_64-cmake + runs-on: faiss-amd-MI200 + container: + image: ubuntu:22.04 + options: --device=/dev/kfd --device=/dev/dri --ipc=host --shm-size 16G --group-add video --cap-add=SYS_PTRACE --cap-add=SYS_ADMIN + steps: + - name: Container setup + run: | + if [ -f /.dockerenv ]; then + apt-get update && apt-get install -y sudo && apt-get install -y git + git config --global --add safe.directory '*' + else + echo 'Skipping. Current job is not running inside a container.' + fi + - name: Checkout + uses: actions/checkout@v4 + - name: Build and Test (cmake) + uses: ./.github/actions/build_cmake + with: + gpu: ON + rocm: ON + linux-arm64-SVE-cmake: + name: Linux arm64 SVE (cmake) + needs: linux-x86_64-cmake + runs-on: faiss-aws-r8g.large + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Build and Test (cmake) + uses: ./.github/actions/build_cmake + with: + opt_level: sve + env: + # Context: https://github.com/facebookresearch/faiss/wiki/Troubleshooting#surprising-faiss-openmp-and-openblas-interaction + OPENBLAS_NUM_THREADS: '1' + linux-x86_64-conda: + name: Linux x86_64 (conda) + needs: linux-x86_64-cmake + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + - name: Build and Package (conda) + uses: ./.github/actions/build_conda + windows-x86_64-conda: + name: Windows x86_64 (conda) + needs: linux-x86_64-cmake + runs-on: windows-2019 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + - name: Build and Package (conda) + uses: ./.github/actions/build_conda + linux-arm64-conda: + name: Linux arm64 (conda) + needs: linux-x86_64-cmake + runs-on: 2-core-ubuntu-arm + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + - name: Build and Package (conda) + uses: ./.github/actions/build_conda diff --git a/.github/workflows/build-release.yml b/.github/workflows/build-release.yml new file mode 100644 index 0000000000..865c9afb25 --- /dev/null +++ b/.github/workflows/build-release.yml @@ -0,0 +1,148 @@ +name: Build Release +on: + workflow_dispatch: + pull_request: + branches: + - main + push: + tags: + - 'v*' +env: + OMP_NUM_THREADS: '10' + MKL_THREADING_LAYER: GNU +jobs: + linux-x86_64-packages: + name: Linux x86_64 packages + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + - name: Build and Package (conda) + uses: ./.github/actions/build_conda + env: + ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} + with: + label: main + linux-x86_64-GPU-packages-CUDA-11-4-4: + name: Linux x86_64 GPU packages (CUDA 11.4.4) + runs-on: 4-core-ubuntu-gpu-t4 + env: + CUDA_ARCHS: "60-real;61-real;62-real;70-real;72-real;75-real;80;86-real" + FAISS_FLATTEN_CONDA_INCLUDES: "1" + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + - name: Build and Package (conda) + uses: ./.github/actions/build_conda + env: + ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} + with: + label: main + cuda: "11.4.4" + linux-x86_64-GPU-CUVS-packages-CUDA11-8-0: + name: Linux x86_64 GPU w/ cuVS packages (CUDA 11.8.0) + runs-on: 4-core-ubuntu-gpu-t4 + env: + CUDA_ARCHS: "70-real;72-real;75-real;80;86-real" + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + - name: Build and Package (conda) + uses: ./.github/actions/build_conda + env: + ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} + with: + label: main + cuvs: "ON" + cuda: "11.8.0" + linux-x86_64-GPU-packages-CUDA-12-1-1: + name: Linux x86_64 GPU packages (CUDA 12.1.1) + runs-on: 4-core-ubuntu-gpu-t4 + env: + CUDA_ARCHS: "70-real;72-real;75-real;80;86-real" + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + - name: Build and Package (conda) + uses: ./.github/actions/build_conda + env: + ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} + with: + label: main + cuda: "12.1.1" + linux-x86_64-GPU-CUVS-packages-CUDA12-4-0: + name: Linux x86_64 GPU w/ cuVS packages (CUDA 12.4.0) + runs-on: 4-core-ubuntu-gpu-t4 + env: + CUDA_ARCHS: "70-real;72-real;75-real;80;86-real" + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + - name: Build and Package (conda) + uses: ./.github/actions/build_conda + env: + ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} + with: + label: main + cuvs: "ON" + cuda: "12.4.0" + windows-x86_64-packages: + name: Windows x86_64 packages + runs-on: windows-2019 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + - name: Build and Package (conda) + uses: ./.github/actions/build_conda + env: + ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} + with: + label: main + osx-arm64-packages: + name: OSX arm64 packages + runs-on: macos-14 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + - name: Build and Package (conda) + uses: ./.github/actions/build_conda + env: + ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} + with: + label: main + linux-arm64-packages: + name: Linux arm64 packages + runs-on: 2-core-ubuntu-arm + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + - name: Build and Package (conda) + uses: ./.github/actions/build_conda + env: + ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} + with: + label: main diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4d38b583d6..54986039c2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -7,302 +7,10 @@ on: push: tags: - 'v*' -env: - OMP_NUM_THREADS: '10' - MKL_THREADING_LAYER: GNU jobs: - format: - name: Format - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Install clang-format - run: | - sudo apt-get update -y - sudo apt-get install -y wget - sudo apt install -y lsb-release wget software-properties-common gnupg - wget https://apt.llvm.org/llvm.sh - chmod u+x llvm.sh - sudo ./llvm.sh 18 - sudo apt-get install -y git-core clang-format-18 - - name: Verify clang-format - run: | - git ls-files | grep -E '\.(cpp|h|cu|cuh)$' | xargs clang-format-18 -i - if git diff --quiet; then - echo "Formatting OK!" - else - echo "Formatting not OK!" - echo "------------------" - git --no-pager diff --color - exit 1 - fi - linux-x86_64-cmake: - name: Linux x86_64 (cmake) - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Build and Test (cmake) - uses: ./.github/actions/build_cmake - linux-x86_64-AVX2-cmake: - name: Linux x86_64 AVX2 (cmake) - needs: linux-x86_64-cmake - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Build and Test (cmake) - uses: ./.github/actions/build_cmake - with: - opt_level: avx2 - linux-x86_64-AVX512-cmake: - name: Linux x86_64 AVX512 (cmake) - needs: linux-x86_64-cmake - runs-on: faiss-aws-m7i.large - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Build and Test (cmake) - uses: ./.github/actions/build_cmake - with: - opt_level: avx512 - linux-x86_64-GPU-cmake: - name: Linux x86_64 GPU (cmake) - needs: linux-x86_64-cmake - runs-on: 4-core-ubuntu-gpu-t4 - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Build and Test (cmake) - uses: ./.github/actions/build_cmake - with: - gpu: ON - linux-x86_64-GPU-w-CUVS-cmake: - name: Linux x86_64 GPU w/ cuVS (cmake) - needs: linux-x86_64-cmake - runs-on: 4-core-ubuntu-gpu-t4 - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Build and Test (cmake) - uses: ./.github/actions/build_cmake - with: - gpu: ON - cuvs: ON - linux-x86_64-GPU-w-ROCm-cmake: - name: Linux x86_64 GPU w/ ROCm (cmake) - needs: linux-x86_64-cmake - runs-on: faiss-amd-MI200 - container: - image: ubuntu:22.04 - options: --device=/dev/kfd --device=/dev/dri --ipc=host --shm-size 16G --group-add video --cap-add=SYS_PTRACE --cap-add=SYS_ADMIN - steps: - - name: Container setup - run: | - if [ -f /.dockerenv ]; then - apt-get update && apt-get install -y sudo && apt-get install -y git - git config --global --add safe.directory '*' - else - echo 'Skipping. Current job is not running inside a container.' - fi - - name: Checkout - uses: actions/checkout@v4 - - name: Build and Test (cmake) - uses: ./.github/actions/build_cmake - with: - gpu: ON - rocm: ON - linux-arm64-SVE-cmake: - name: Linux arm64 SVE (cmake) - needs: linux-x86_64-cmake - runs-on: faiss-aws-r8g.large - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Build and Test (cmake) - uses: ./.github/actions/build_cmake - with: - opt_level: sve - env: - # Context: https://github.com/facebookresearch/faiss/wiki/Troubleshooting#surprising-faiss-openmp-and-openblas-interaction - OPENBLAS_NUM_THREADS: '1' - linux-x86_64-conda: - name: Linux x86_64 (conda) - needs: linux-x86_64-cmake - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - - name: Build and Package (conda) - uses: ./.github/actions/build_conda - windows-x86_64-conda: - name: Windows x86_64 (conda) - needs: linux-x86_64-cmake - runs-on: windows-2019 - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - - name: Build and Package (conda) - uses: ./.github/actions/build_conda - linux-arm64-conda: - name: Linux arm64 (conda) - needs: linux-x86_64-cmake - runs-on: 2-core-ubuntu-arm - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - - name: Build and Package (conda) - uses: ./.github/actions/build_conda - linux-x86_64-packages: - name: Linux x86_64 packages + build-pull-request: + uses: ./build-pull-request.yml + build-release: + uses: ./build-pull-request.yml + uses: ./build-release.yml if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - - name: Build and Package (conda) - uses: ./.github/actions/build_conda - env: - ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} - with: - label: main - linux-x86_64-GPU-packages-CUDA-11-4-4: - name: Linux x86_64 GPU packages (CUDA 11.4.4) - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') - runs-on: 4-core-ubuntu-gpu-t4 - env: - CUDA_ARCHS: "60-real;61-real;62-real;70-real;72-real;75-real;80;86-real" - FAISS_FLATTEN_CONDA_INCLUDES: "1" - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - - name: Build and Package (conda) - uses: ./.github/actions/build_conda - env: - ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} - with: - label: main - cuda: "11.4.4" - linux-x86_64-GPU-CUVS-packages-CUDA11-8-0: - name: Linux x86_64 GPU w/ cuVS packages (CUDA 11.8.0) - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') - runs-on: 4-core-ubuntu-gpu-t4 - env: - CUDA_ARCHS: "70-real;72-real;75-real;80;86-real" - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - - name: Build and Package (conda) - uses: ./.github/actions/build_conda - env: - ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} - with: - label: main - cuvs: "ON" - cuda: "11.8.0" - linux-x86_64-GPU-packages-CUDA-12-1-1: - name: Linux x86_64 GPU packages (CUDA 12.1.1) - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') - runs-on: 4-core-ubuntu-gpu-t4 - env: - CUDA_ARCHS: "70-real;72-real;75-real;80;86-real" - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - - name: Build and Package (conda) - uses: ./.github/actions/build_conda - env: - ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} - with: - label: main - cuda: "12.1.1" - linux-x86_64-GPU-CUVS-packages-CUDA12-4-0: - name: Linux x86_64 GPU w/ cuVS packages (CUDA 12.4.0) - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') - runs-on: 4-core-ubuntu-gpu-t4 - env: - CUDA_ARCHS: "70-real;72-real;75-real;80;86-real" - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - - name: Build and Package (conda) - uses: ./.github/actions/build_conda - env: - ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} - with: - label: main - cuvs: "ON" - cuda: "12.4.0" - windows-x86_64-packages: - name: Windows x86_64 packages - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') - runs-on: windows-2019 - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - - name: Build and Package (conda) - uses: ./.github/actions/build_conda - env: - ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} - with: - label: main - osx-arm64-packages: - name: OSX arm64 packages - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') - runs-on: macos-14 - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - - name: Build and Package (conda) - uses: ./.github/actions/build_conda - env: - ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} - with: - label: main - linux-arm64-packages: - name: Linux arm64 packages - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') - runs-on: 2-core-ubuntu-arm - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - - name: Build and Package (conda) - uses: ./.github/actions/build_conda - env: - ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} - with: - label: main From 16e1645ab3c1e482f705411b1e72ba7519562205 Mon Sep 17 00:00:00 2001 From: Junjie Qi Date: Thu, 5 Dec 2024 13:37:31 -0800 Subject: [PATCH 02/20] change build name --- .github/workflows/build-pull-request.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-pull-request.yml b/.github/workflows/build-pull-request.yml index c8372fa34c..7f9e944e5f 100644 --- a/.github/workflows/build-pull-request.yml +++ b/.github/workflows/build-pull-request.yml @@ -1,4 +1,4 @@ -name: Build +name: Build Pull Request on: workflow_dispatch: pull_request: From 17537a3fbd67cc1905af67db17145f31e31a3673 Mon Sep 17 00:00:00 2001 From: Junjie Qi Date: Thu, 5 Dec 2024 14:42:24 -0800 Subject: [PATCH 03/20] hide release and pr workflow --- .github/workflows/build-pull-request.yml | 165 ----------------------- .github/workflows/build-release.yml | 148 -------------------- .github/workflows/build.yml | 6 +- 3 files changed, 3 insertions(+), 316 deletions(-) delete mode 100644 .github/workflows/build-pull-request.yml delete mode 100644 .github/workflows/build-release.yml diff --git a/.github/workflows/build-pull-request.yml b/.github/workflows/build-pull-request.yml deleted file mode 100644 index 7f9e944e5f..0000000000 --- a/.github/workflows/build-pull-request.yml +++ /dev/null @@ -1,165 +0,0 @@ -name: Build Pull Request -on: - workflow_dispatch: - pull_request: - branches: - - main - push: - tags: - - 'v*' -env: - OMP_NUM_THREADS: '10' - MKL_THREADING_LAYER: GNU -jobs: - format: - name: Format - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Install clang-format - run: | - sudo apt-get update -y - sudo apt-get install -y wget - sudo apt install -y lsb-release wget software-properties-common gnupg - wget https://apt.llvm.org/llvm.sh - chmod u+x llvm.sh - sudo ./llvm.sh 18 - sudo apt-get install -y git-core clang-format-18 - - name: Verify clang-format - run: | - git ls-files | grep -E '\.(cpp|h|cu|cuh)$' | xargs clang-format-18 -i - if git diff --quiet; then - echo "Formatting OK!" - else - echo "Formatting not OK!" - echo "------------------" - git --no-pager diff --color - exit 1 - fi - linux-x86_64-cmake: - name: Linux x86_64 (cmake) - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Build and Test (cmake) - uses: ./.github/actions/build_cmake - linux-x86_64-AVX2-cmake: - name: Linux x86_64 AVX2 (cmake) - needs: linux-x86_64-cmake - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Build and Test (cmake) - uses: ./.github/actions/build_cmake - with: - opt_level: avx2 - linux-x86_64-AVX512-cmake: - name: Linux x86_64 AVX512 (cmake) - needs: linux-x86_64-cmake - runs-on: faiss-aws-m7i.large - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Build and Test (cmake) - uses: ./.github/actions/build_cmake - with: - opt_level: avx512 - linux-x86_64-GPU-cmake: - name: Linux x86_64 GPU (cmake) - needs: linux-x86_64-cmake - runs-on: 4-core-ubuntu-gpu-t4 - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Build and Test (cmake) - uses: ./.github/actions/build_cmake - with: - gpu: ON - linux-x86_64-GPU-w-CUVS-cmake: - name: Linux x86_64 GPU w/ cuVS (cmake) - needs: linux-x86_64-cmake - runs-on: 4-core-ubuntu-gpu-t4 - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Build and Test (cmake) - uses: ./.github/actions/build_cmake - with: - gpu: ON - cuvs: ON - linux-x86_64-GPU-w-ROCm-cmake: - name: Linux x86_64 GPU w/ ROCm (cmake) - needs: linux-x86_64-cmake - runs-on: faiss-amd-MI200 - container: - image: ubuntu:22.04 - options: --device=/dev/kfd --device=/dev/dri --ipc=host --shm-size 16G --group-add video --cap-add=SYS_PTRACE --cap-add=SYS_ADMIN - steps: - - name: Container setup - run: | - if [ -f /.dockerenv ]; then - apt-get update && apt-get install -y sudo && apt-get install -y git - git config --global --add safe.directory '*' - else - echo 'Skipping. Current job is not running inside a container.' - fi - - name: Checkout - uses: actions/checkout@v4 - - name: Build and Test (cmake) - uses: ./.github/actions/build_cmake - with: - gpu: ON - rocm: ON - linux-arm64-SVE-cmake: - name: Linux arm64 SVE (cmake) - needs: linux-x86_64-cmake - runs-on: faiss-aws-r8g.large - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Build and Test (cmake) - uses: ./.github/actions/build_cmake - with: - opt_level: sve - env: - # Context: https://github.com/facebookresearch/faiss/wiki/Troubleshooting#surprising-faiss-openmp-and-openblas-interaction - OPENBLAS_NUM_THREADS: '1' - linux-x86_64-conda: - name: Linux x86_64 (conda) - needs: linux-x86_64-cmake - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - - name: Build and Package (conda) - uses: ./.github/actions/build_conda - windows-x86_64-conda: - name: Windows x86_64 (conda) - needs: linux-x86_64-cmake - runs-on: windows-2019 - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - - name: Build and Package (conda) - uses: ./.github/actions/build_conda - linux-arm64-conda: - name: Linux arm64 (conda) - needs: linux-x86_64-cmake - runs-on: 2-core-ubuntu-arm - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - - name: Build and Package (conda) - uses: ./.github/actions/build_conda diff --git a/.github/workflows/build-release.yml b/.github/workflows/build-release.yml deleted file mode 100644 index 865c9afb25..0000000000 --- a/.github/workflows/build-release.yml +++ /dev/null @@ -1,148 +0,0 @@ -name: Build Release -on: - workflow_dispatch: - pull_request: - branches: - - main - push: - tags: - - 'v*' -env: - OMP_NUM_THREADS: '10' - MKL_THREADING_LAYER: GNU -jobs: - linux-x86_64-packages: - name: Linux x86_64 packages - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - - name: Build and Package (conda) - uses: ./.github/actions/build_conda - env: - ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} - with: - label: main - linux-x86_64-GPU-packages-CUDA-11-4-4: - name: Linux x86_64 GPU packages (CUDA 11.4.4) - runs-on: 4-core-ubuntu-gpu-t4 - env: - CUDA_ARCHS: "60-real;61-real;62-real;70-real;72-real;75-real;80;86-real" - FAISS_FLATTEN_CONDA_INCLUDES: "1" - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - - name: Build and Package (conda) - uses: ./.github/actions/build_conda - env: - ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} - with: - label: main - cuda: "11.4.4" - linux-x86_64-GPU-CUVS-packages-CUDA11-8-0: - name: Linux x86_64 GPU w/ cuVS packages (CUDA 11.8.0) - runs-on: 4-core-ubuntu-gpu-t4 - env: - CUDA_ARCHS: "70-real;72-real;75-real;80;86-real" - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - - name: Build and Package (conda) - uses: ./.github/actions/build_conda - env: - ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} - with: - label: main - cuvs: "ON" - cuda: "11.8.0" - linux-x86_64-GPU-packages-CUDA-12-1-1: - name: Linux x86_64 GPU packages (CUDA 12.1.1) - runs-on: 4-core-ubuntu-gpu-t4 - env: - CUDA_ARCHS: "70-real;72-real;75-real;80;86-real" - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - - name: Build and Package (conda) - uses: ./.github/actions/build_conda - env: - ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} - with: - label: main - cuda: "12.1.1" - linux-x86_64-GPU-CUVS-packages-CUDA12-4-0: - name: Linux x86_64 GPU w/ cuVS packages (CUDA 12.4.0) - runs-on: 4-core-ubuntu-gpu-t4 - env: - CUDA_ARCHS: "70-real;72-real;75-real;80;86-real" - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - - name: Build and Package (conda) - uses: ./.github/actions/build_conda - env: - ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} - with: - label: main - cuvs: "ON" - cuda: "12.4.0" - windows-x86_64-packages: - name: Windows x86_64 packages - runs-on: windows-2019 - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - - name: Build and Package (conda) - uses: ./.github/actions/build_conda - env: - ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} - with: - label: main - osx-arm64-packages: - name: OSX arm64 packages - runs-on: macos-14 - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - - name: Build and Package (conda) - uses: ./.github/actions/build_conda - env: - ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} - with: - label: main - linux-arm64-packages: - name: Linux arm64 packages - runs-on: 2-core-ubuntu-arm - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - - name: Build and Package (conda) - uses: ./.github/actions/build_conda - env: - ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} - with: - label: main diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 54986039c2..65f3d78536 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,8 +9,8 @@ on: - 'v*' jobs: build-pull-request: - uses: ./build-pull-request.yml + uses: ./internal/build-pull-request.yml build-release: - uses: ./build-pull-request.yml - uses: ./build-release.yml + uses: ./internal/build-pull-request.yml + uses: ./internal/build-release.yml if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') From 366d44a6aa1fbfe04311baabe0eb7902fb213b89 Mon Sep 17 00:00:00 2001 From: Junjie Qi Date: Thu, 5 Dec 2024 14:47:18 -0800 Subject: [PATCH 04/20] hide release and pr workflow --- .github/workflows/build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 65f3d78536..b6b096fb0a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,8 +9,8 @@ on: - 'v*' jobs: build-pull-request: - uses: ./internal/build-pull-request.yml + uses: ./.github/workflows/internal/build-pull-request.yml build-release: - uses: ./internal/build-pull-request.yml - uses: ./internal/build-release.yml + uses: ./.github/workflows/internal/build-pull-request.yml + uses: ./.github/workflows/internal/build-release.yml if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') From 4b4b3426ae21043d0debad6803c8351e4082a701 Mon Sep 17 00:00:00 2001 From: Junjie Qi Date: Thu, 5 Dec 2024 15:12:15 -0800 Subject: [PATCH 05/20] add condition in the release first --- .github/workflows/build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b6b096fb0a..54986039c2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,8 +9,8 @@ on: - 'v*' jobs: build-pull-request: - uses: ./.github/workflows/internal/build-pull-request.yml + uses: ./build-pull-request.yml build-release: - uses: ./.github/workflows/internal/build-pull-request.yml - uses: ./.github/workflows/internal/build-release.yml + uses: ./build-pull-request.yml + uses: ./build-release.yml if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') From 7b081eafa8223e40d91911cd8bd0d2e2dde5ec73 Mon Sep 17 00:00:00 2001 From: Junjie Qi Date: Thu, 5 Dec 2024 15:27:22 -0800 Subject: [PATCH 06/20] add condition in the release first --- .github/workflows/build-pull-request.yml | 165 +++++++++++++++++++++++ .github/workflows/build-release.yml | 156 +++++++++++++++++++++ 2 files changed, 321 insertions(+) create mode 100644 .github/workflows/build-pull-request.yml create mode 100644 .github/workflows/build-release.yml diff --git a/.github/workflows/build-pull-request.yml b/.github/workflows/build-pull-request.yml new file mode 100644 index 0000000000..7f9e944e5f --- /dev/null +++ b/.github/workflows/build-pull-request.yml @@ -0,0 +1,165 @@ +name: Build Pull Request +on: + workflow_dispatch: + pull_request: + branches: + - main + push: + tags: + - 'v*' +env: + OMP_NUM_THREADS: '10' + MKL_THREADING_LAYER: GNU +jobs: + format: + name: Format + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Install clang-format + run: | + sudo apt-get update -y + sudo apt-get install -y wget + sudo apt install -y lsb-release wget software-properties-common gnupg + wget https://apt.llvm.org/llvm.sh + chmod u+x llvm.sh + sudo ./llvm.sh 18 + sudo apt-get install -y git-core clang-format-18 + - name: Verify clang-format + run: | + git ls-files | grep -E '\.(cpp|h|cu|cuh)$' | xargs clang-format-18 -i + if git diff --quiet; then + echo "Formatting OK!" + else + echo "Formatting not OK!" + echo "------------------" + git --no-pager diff --color + exit 1 + fi + linux-x86_64-cmake: + name: Linux x86_64 (cmake) + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Build and Test (cmake) + uses: ./.github/actions/build_cmake + linux-x86_64-AVX2-cmake: + name: Linux x86_64 AVX2 (cmake) + needs: linux-x86_64-cmake + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Build and Test (cmake) + uses: ./.github/actions/build_cmake + with: + opt_level: avx2 + linux-x86_64-AVX512-cmake: + name: Linux x86_64 AVX512 (cmake) + needs: linux-x86_64-cmake + runs-on: faiss-aws-m7i.large + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Build and Test (cmake) + uses: ./.github/actions/build_cmake + with: + opt_level: avx512 + linux-x86_64-GPU-cmake: + name: Linux x86_64 GPU (cmake) + needs: linux-x86_64-cmake + runs-on: 4-core-ubuntu-gpu-t4 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Build and Test (cmake) + uses: ./.github/actions/build_cmake + with: + gpu: ON + linux-x86_64-GPU-w-CUVS-cmake: + name: Linux x86_64 GPU w/ cuVS (cmake) + needs: linux-x86_64-cmake + runs-on: 4-core-ubuntu-gpu-t4 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Build and Test (cmake) + uses: ./.github/actions/build_cmake + with: + gpu: ON + cuvs: ON + linux-x86_64-GPU-w-ROCm-cmake: + name: Linux x86_64 GPU w/ ROCm (cmake) + needs: linux-x86_64-cmake + runs-on: faiss-amd-MI200 + container: + image: ubuntu:22.04 + options: --device=/dev/kfd --device=/dev/dri --ipc=host --shm-size 16G --group-add video --cap-add=SYS_PTRACE --cap-add=SYS_ADMIN + steps: + - name: Container setup + run: | + if [ -f /.dockerenv ]; then + apt-get update && apt-get install -y sudo && apt-get install -y git + git config --global --add safe.directory '*' + else + echo 'Skipping. Current job is not running inside a container.' + fi + - name: Checkout + uses: actions/checkout@v4 + - name: Build and Test (cmake) + uses: ./.github/actions/build_cmake + with: + gpu: ON + rocm: ON + linux-arm64-SVE-cmake: + name: Linux arm64 SVE (cmake) + needs: linux-x86_64-cmake + runs-on: faiss-aws-r8g.large + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Build and Test (cmake) + uses: ./.github/actions/build_cmake + with: + opt_level: sve + env: + # Context: https://github.com/facebookresearch/faiss/wiki/Troubleshooting#surprising-faiss-openmp-and-openblas-interaction + OPENBLAS_NUM_THREADS: '1' + linux-x86_64-conda: + name: Linux x86_64 (conda) + needs: linux-x86_64-cmake + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + - name: Build and Package (conda) + uses: ./.github/actions/build_conda + windows-x86_64-conda: + name: Windows x86_64 (conda) + needs: linux-x86_64-cmake + runs-on: windows-2019 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + - name: Build and Package (conda) + uses: ./.github/actions/build_conda + linux-arm64-conda: + name: Linux arm64 (conda) + needs: linux-x86_64-cmake + runs-on: 2-core-ubuntu-arm + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + - name: Build and Package (conda) + uses: ./.github/actions/build_conda diff --git a/.github/workflows/build-release.yml b/.github/workflows/build-release.yml new file mode 100644 index 0000000000..60b2358fe7 --- /dev/null +++ b/.github/workflows/build-release.yml @@ -0,0 +1,156 @@ +name: Build Release +on: + workflow_dispatch: + pull_request: + branches: + - main + push: + tags: + - 'v*' +env: + OMP_NUM_THREADS: '10' + MKL_THREADING_LAYER: GNU +jobs: + linux-x86_64-packages: + name: Linux x86_64 packages + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + - name: Build and Package (conda) + uses: ./.github/actions/build_conda + env: + ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} + with: + label: main + linux-x86_64-GPU-packages-CUDA-11-4-4: + name: Linux x86_64 GPU packages (CUDA 11.4.4) + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') + runs-on: 4-core-ubuntu-gpu-t4 + env: + CUDA_ARCHS: "60-real;61-real;62-real;70-real;72-real;75-real;80;86-real" + FAISS_FLATTEN_CONDA_INCLUDES: "1" + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + - name: Build and Package (conda) + uses: ./.github/actions/build_conda + env: + ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} + with: + label: main + cuda: "11.4.4" + linux-x86_64-GPU-CUVS-packages-CUDA11-8-0: + name: Linux x86_64 GPU w/ cuVS packages (CUDA 11.8.0) + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') + runs-on: 4-core-ubuntu-gpu-t4 + env: + CUDA_ARCHS: "70-real;72-real;75-real;80;86-real" + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + - name: Build and Package (conda) + uses: ./.github/actions/build_conda + env: + ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} + with: + label: main + cuvs: "ON" + cuda: "11.8.0" + linux-x86_64-GPU-packages-CUDA-12-1-1: + name: Linux x86_64 GPU packages (CUDA 12.1.1) + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') + runs-on: 4-core-ubuntu-gpu-t4 + env: + CUDA_ARCHS: "70-real;72-real;75-real;80;86-real" + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + - name: Build and Package (conda) + uses: ./.github/actions/build_conda + env: + ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} + with: + label: main + cuda: "12.1.1" + linux-x86_64-GPU-CUVS-packages-CUDA12-4-0: + name: Linux x86_64 GPU w/ cuVS packages (CUDA 12.4.0) + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') + runs-on: 4-core-ubuntu-gpu-t4 + env: + CUDA_ARCHS: "70-real;72-real;75-real;80;86-real" + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + - name: Build and Package (conda) + uses: ./.github/actions/build_conda + env: + ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} + with: + label: main + cuvs: "ON" + cuda: "12.4.0" + windows-x86_64-packages: + name: Windows x86_64 packages + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') + runs-on: windows-2019 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + - name: Build and Package (conda) + uses: ./.github/actions/build_conda + env: + ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} + with: + label: main + osx-arm64-packages: + name: OSX arm64 packages + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') + runs-on: macos-14 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + - name: Build and Package (conda) + uses: ./.github/actions/build_conda + env: + ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} + with: + label: main + linux-arm64-packages: + name: Linux arm64 packages + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') + runs-on: 2-core-ubuntu-arm + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + - name: Build and Package (conda) + uses: ./.github/actions/build_conda + env: + ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} + with: + label: main From 8cc595d3fe97bbeed6ae166c2668b246222c76d8 Mon Sep 17 00:00:00 2001 From: Junjie Qi Date: Thu, 5 Dec 2024 16:30:21 -0800 Subject: [PATCH 07/20] add condition in the release first --- .github/workflows/build-pull-request.yml | 8 +------- .github/workflows/build-release.yml | 8 +------- .github/workflows/build.yml | 1 - 3 files changed, 2 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build-pull-request.yml b/.github/workflows/build-pull-request.yml index 7f9e944e5f..e5f3e6614d 100644 --- a/.github/workflows/build-pull-request.yml +++ b/.github/workflows/build-pull-request.yml @@ -1,12 +1,6 @@ name: Build Pull Request on: - workflow_dispatch: - pull_request: - branches: - - main - push: - tags: - - 'v*' + workflow_call: env: OMP_NUM_THREADS: '10' MKL_THREADING_LAYER: GNU diff --git a/.github/workflows/build-release.yml b/.github/workflows/build-release.yml index 60b2358fe7..7d0b5b0dfb 100644 --- a/.github/workflows/build-release.yml +++ b/.github/workflows/build-release.yml @@ -1,12 +1,6 @@ name: Build Release on: - workflow_dispatch: - pull_request: - branches: - - main - push: - tags: - - 'v*' + workflow_call: env: OMP_NUM_THREADS: '10' MKL_THREADING_LAYER: GNU diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 54986039c2..75acab231f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -11,6 +11,5 @@ jobs: build-pull-request: uses: ./build-pull-request.yml build-release: - uses: ./build-pull-request.yml uses: ./build-release.yml if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') From c8a83285658797f428fa89908203a507a6496e08 Mon Sep 17 00:00:00 2001 From: Junjie Qi Date: Thu, 5 Dec 2024 16:33:02 -0800 Subject: [PATCH 08/20] try to fix the commit --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 75acab231f..cfaddecb9f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,7 +9,7 @@ on: - 'v*' jobs: build-pull-request: - uses: ./build-pull-request.yml + uses: ./build-pull-request.yml@ build-release: - uses: ./build-release.yml + uses: ./build-release.yml@ if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') From 6bb2474a889c48acb462a1373ec28571ddaa7b1b Mon Sep 17 00:00:00 2001 From: Junjie Qi Date: Thu, 5 Dec 2024 16:34:46 -0800 Subject: [PATCH 09/20] try to fix the commit --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cfaddecb9f..37b1803678 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,7 +9,7 @@ on: - 'v*' jobs: build-pull-request: - uses: ./build-pull-request.yml@ + uses: ./build-pull-request.yml@separate_workflow build-release: - uses: ./build-release.yml@ + uses: ./build-release.yml@separate_workflow if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') From 0e9cc6c7383198b0531fe602c830b849fd2a2316 Mon Sep 17 00:00:00 2001 From: Junjie Qi Date: Thu, 5 Dec 2024 16:35:59 -0800 Subject: [PATCH 10/20] try to fix the commit --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 37b1803678..10318d71e6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,7 +9,7 @@ on: - 'v*' jobs: build-pull-request: - uses: ./build-pull-request.yml@separate_workflow + uses: .github/workflows/build-pull-request.yml@separate_workflow build-release: - uses: ./build-release.yml@separate_workflow + uses: .github/workflows/build-release.yml@separate_workflow if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') From 2a7fd3a86156f6ea49288746cb8d704e19868e14 Mon Sep 17 00:00:00 2001 From: Junjie Qi Date: Thu, 5 Dec 2024 16:36:21 -0800 Subject: [PATCH 11/20] try to fix the commit --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 10318d71e6..8ac46055f6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,7 +9,7 @@ on: - 'v*' jobs: build-pull-request: - uses: .github/workflows/build-pull-request.yml@separate_workflow + uses: ./.github/workflows/build-pull-request.yml@separate_workflow build-release: - uses: .github/workflows/build-release.yml@separate_workflow + uses: ./.github/workflows/build-release.yml@separate_workflow if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') From dc742386214736ee9c9997626168eddbd76c0787 Mon Sep 17 00:00:00 2001 From: Junjie Qi Date: Thu, 5 Dec 2024 16:37:09 -0800 Subject: [PATCH 12/20] try to fix the commit --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8ac46055f6..d9f2f8334d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,7 +9,7 @@ on: - 'v*' jobs: build-pull-request: - uses: ./.github/workflows/build-pull-request.yml@separate_workflow + uses: ./.github/workflows/build-pull-request.yml build-release: - uses: ./.github/workflows/build-release.yml@separate_workflow + uses: ./.github/workflows/build-release.yml if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') From d417a49753af28648c685e242cbc3ef7125878c3 Mon Sep 17 00:00:00 2001 From: Junjie Qi Date: Thu, 5 Dec 2024 16:38:55 -0800 Subject: [PATCH 13/20] try to fix the commit --- .github/workflows/build-pull-request.yml | 159 ----------------------- .github/workflows/build-release.yml | 150 --------------------- .github/workflows/build.yml | 4 +- 3 files changed, 2 insertions(+), 311 deletions(-) delete mode 100644 .github/workflows/build-pull-request.yml delete mode 100644 .github/workflows/build-release.yml diff --git a/.github/workflows/build-pull-request.yml b/.github/workflows/build-pull-request.yml deleted file mode 100644 index e5f3e6614d..0000000000 --- a/.github/workflows/build-pull-request.yml +++ /dev/null @@ -1,159 +0,0 @@ -name: Build Pull Request -on: - workflow_call: -env: - OMP_NUM_THREADS: '10' - MKL_THREADING_LAYER: GNU -jobs: - format: - name: Format - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Install clang-format - run: | - sudo apt-get update -y - sudo apt-get install -y wget - sudo apt install -y lsb-release wget software-properties-common gnupg - wget https://apt.llvm.org/llvm.sh - chmod u+x llvm.sh - sudo ./llvm.sh 18 - sudo apt-get install -y git-core clang-format-18 - - name: Verify clang-format - run: | - git ls-files | grep -E '\.(cpp|h|cu|cuh)$' | xargs clang-format-18 -i - if git diff --quiet; then - echo "Formatting OK!" - else - echo "Formatting not OK!" - echo "------------------" - git --no-pager diff --color - exit 1 - fi - linux-x86_64-cmake: - name: Linux x86_64 (cmake) - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Build and Test (cmake) - uses: ./.github/actions/build_cmake - linux-x86_64-AVX2-cmake: - name: Linux x86_64 AVX2 (cmake) - needs: linux-x86_64-cmake - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Build and Test (cmake) - uses: ./.github/actions/build_cmake - with: - opt_level: avx2 - linux-x86_64-AVX512-cmake: - name: Linux x86_64 AVX512 (cmake) - needs: linux-x86_64-cmake - runs-on: faiss-aws-m7i.large - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Build and Test (cmake) - uses: ./.github/actions/build_cmake - with: - opt_level: avx512 - linux-x86_64-GPU-cmake: - name: Linux x86_64 GPU (cmake) - needs: linux-x86_64-cmake - runs-on: 4-core-ubuntu-gpu-t4 - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Build and Test (cmake) - uses: ./.github/actions/build_cmake - with: - gpu: ON - linux-x86_64-GPU-w-CUVS-cmake: - name: Linux x86_64 GPU w/ cuVS (cmake) - needs: linux-x86_64-cmake - runs-on: 4-core-ubuntu-gpu-t4 - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Build and Test (cmake) - uses: ./.github/actions/build_cmake - with: - gpu: ON - cuvs: ON - linux-x86_64-GPU-w-ROCm-cmake: - name: Linux x86_64 GPU w/ ROCm (cmake) - needs: linux-x86_64-cmake - runs-on: faiss-amd-MI200 - container: - image: ubuntu:22.04 - options: --device=/dev/kfd --device=/dev/dri --ipc=host --shm-size 16G --group-add video --cap-add=SYS_PTRACE --cap-add=SYS_ADMIN - steps: - - name: Container setup - run: | - if [ -f /.dockerenv ]; then - apt-get update && apt-get install -y sudo && apt-get install -y git - git config --global --add safe.directory '*' - else - echo 'Skipping. Current job is not running inside a container.' - fi - - name: Checkout - uses: actions/checkout@v4 - - name: Build and Test (cmake) - uses: ./.github/actions/build_cmake - with: - gpu: ON - rocm: ON - linux-arm64-SVE-cmake: - name: Linux arm64 SVE (cmake) - needs: linux-x86_64-cmake - runs-on: faiss-aws-r8g.large - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Build and Test (cmake) - uses: ./.github/actions/build_cmake - with: - opt_level: sve - env: - # Context: https://github.com/facebookresearch/faiss/wiki/Troubleshooting#surprising-faiss-openmp-and-openblas-interaction - OPENBLAS_NUM_THREADS: '1' - linux-x86_64-conda: - name: Linux x86_64 (conda) - needs: linux-x86_64-cmake - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - - name: Build and Package (conda) - uses: ./.github/actions/build_conda - windows-x86_64-conda: - name: Windows x86_64 (conda) - needs: linux-x86_64-cmake - runs-on: windows-2019 - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - - name: Build and Package (conda) - uses: ./.github/actions/build_conda - linux-arm64-conda: - name: Linux arm64 (conda) - needs: linux-x86_64-cmake - runs-on: 2-core-ubuntu-arm - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - - name: Build and Package (conda) - uses: ./.github/actions/build_conda diff --git a/.github/workflows/build-release.yml b/.github/workflows/build-release.yml deleted file mode 100644 index 7d0b5b0dfb..0000000000 --- a/.github/workflows/build-release.yml +++ /dev/null @@ -1,150 +0,0 @@ -name: Build Release -on: - workflow_call: -env: - OMP_NUM_THREADS: '10' - MKL_THREADING_LAYER: GNU -jobs: - linux-x86_64-packages: - name: Linux x86_64 packages - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - - name: Build and Package (conda) - uses: ./.github/actions/build_conda - env: - ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} - with: - label: main - linux-x86_64-GPU-packages-CUDA-11-4-4: - name: Linux x86_64 GPU packages (CUDA 11.4.4) - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') - runs-on: 4-core-ubuntu-gpu-t4 - env: - CUDA_ARCHS: "60-real;61-real;62-real;70-real;72-real;75-real;80;86-real" - FAISS_FLATTEN_CONDA_INCLUDES: "1" - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - - name: Build and Package (conda) - uses: ./.github/actions/build_conda - env: - ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} - with: - label: main - cuda: "11.4.4" - linux-x86_64-GPU-CUVS-packages-CUDA11-8-0: - name: Linux x86_64 GPU w/ cuVS packages (CUDA 11.8.0) - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') - runs-on: 4-core-ubuntu-gpu-t4 - env: - CUDA_ARCHS: "70-real;72-real;75-real;80;86-real" - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - - name: Build and Package (conda) - uses: ./.github/actions/build_conda - env: - ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} - with: - label: main - cuvs: "ON" - cuda: "11.8.0" - linux-x86_64-GPU-packages-CUDA-12-1-1: - name: Linux x86_64 GPU packages (CUDA 12.1.1) - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') - runs-on: 4-core-ubuntu-gpu-t4 - env: - CUDA_ARCHS: "70-real;72-real;75-real;80;86-real" - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - - name: Build and Package (conda) - uses: ./.github/actions/build_conda - env: - ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} - with: - label: main - cuda: "12.1.1" - linux-x86_64-GPU-CUVS-packages-CUDA12-4-0: - name: Linux x86_64 GPU w/ cuVS packages (CUDA 12.4.0) - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') - runs-on: 4-core-ubuntu-gpu-t4 - env: - CUDA_ARCHS: "70-real;72-real;75-real;80;86-real" - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - - name: Build and Package (conda) - uses: ./.github/actions/build_conda - env: - ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} - with: - label: main - cuvs: "ON" - cuda: "12.4.0" - windows-x86_64-packages: - name: Windows x86_64 packages - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') - runs-on: windows-2019 - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - - name: Build and Package (conda) - uses: ./.github/actions/build_conda - env: - ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} - with: - label: main - osx-arm64-packages: - name: OSX arm64 packages - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') - runs-on: macos-14 - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - - name: Build and Package (conda) - uses: ./.github/actions/build_conda - env: - ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} - with: - label: main - linux-arm64-packages: - name: Linux arm64 packages - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') - runs-on: 2-core-ubuntu-arm - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - - name: Build and Package (conda) - uses: ./.github/actions/build_conda - env: - ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} - with: - label: main diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d9f2f8334d..894da5f16f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,7 +9,7 @@ on: - 'v*' jobs: build-pull-request: - uses: ./.github/workflows/build-pull-request.yml + uses: ./.github/workflows/internal/build-pull-request.yml build-release: - uses: ./.github/workflows/build-release.yml + uses: ./.github/workflows/internal/build-release.yml if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') From 6e1f7f14006006c1e44ad5d2cb195b3c4a130037 Mon Sep 17 00:00:00 2001 From: Junjie Qi Date: Thu, 5 Dec 2024 17:04:21 -0800 Subject: [PATCH 14/20] remove condition from release --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 894da5f16f..d9f2f8334d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,7 +9,7 @@ on: - 'v*' jobs: build-pull-request: - uses: ./.github/workflows/internal/build-pull-request.yml + uses: ./.github/workflows/build-pull-request.yml build-release: - uses: ./.github/workflows/internal/build-release.yml + uses: ./.github/workflows/build-release.yml if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') From 776e9d1aa7edbdaf7be579fea1fd99b703fb4913 Mon Sep 17 00:00:00 2001 From: Junjie Qi Date: Thu, 5 Dec 2024 17:05:56 -0800 Subject: [PATCH 15/20] remove condition from release --- .github/workflows/build-pull-request.yml | 159 +++++++++++++++++++++++ .github/workflows/build-release.yml | 142 ++++++++++++++++++++ 2 files changed, 301 insertions(+) create mode 100644 .github/workflows/build-pull-request.yml create mode 100644 .github/workflows/build-release.yml diff --git a/.github/workflows/build-pull-request.yml b/.github/workflows/build-pull-request.yml new file mode 100644 index 0000000000..e5f3e6614d --- /dev/null +++ b/.github/workflows/build-pull-request.yml @@ -0,0 +1,159 @@ +name: Build Pull Request +on: + workflow_call: +env: + OMP_NUM_THREADS: '10' + MKL_THREADING_LAYER: GNU +jobs: + format: + name: Format + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Install clang-format + run: | + sudo apt-get update -y + sudo apt-get install -y wget + sudo apt install -y lsb-release wget software-properties-common gnupg + wget https://apt.llvm.org/llvm.sh + chmod u+x llvm.sh + sudo ./llvm.sh 18 + sudo apt-get install -y git-core clang-format-18 + - name: Verify clang-format + run: | + git ls-files | grep -E '\.(cpp|h|cu|cuh)$' | xargs clang-format-18 -i + if git diff --quiet; then + echo "Formatting OK!" + else + echo "Formatting not OK!" + echo "------------------" + git --no-pager diff --color + exit 1 + fi + linux-x86_64-cmake: + name: Linux x86_64 (cmake) + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Build and Test (cmake) + uses: ./.github/actions/build_cmake + linux-x86_64-AVX2-cmake: + name: Linux x86_64 AVX2 (cmake) + needs: linux-x86_64-cmake + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Build and Test (cmake) + uses: ./.github/actions/build_cmake + with: + opt_level: avx2 + linux-x86_64-AVX512-cmake: + name: Linux x86_64 AVX512 (cmake) + needs: linux-x86_64-cmake + runs-on: faiss-aws-m7i.large + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Build and Test (cmake) + uses: ./.github/actions/build_cmake + with: + opt_level: avx512 + linux-x86_64-GPU-cmake: + name: Linux x86_64 GPU (cmake) + needs: linux-x86_64-cmake + runs-on: 4-core-ubuntu-gpu-t4 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Build and Test (cmake) + uses: ./.github/actions/build_cmake + with: + gpu: ON + linux-x86_64-GPU-w-CUVS-cmake: + name: Linux x86_64 GPU w/ cuVS (cmake) + needs: linux-x86_64-cmake + runs-on: 4-core-ubuntu-gpu-t4 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Build and Test (cmake) + uses: ./.github/actions/build_cmake + with: + gpu: ON + cuvs: ON + linux-x86_64-GPU-w-ROCm-cmake: + name: Linux x86_64 GPU w/ ROCm (cmake) + needs: linux-x86_64-cmake + runs-on: faiss-amd-MI200 + container: + image: ubuntu:22.04 + options: --device=/dev/kfd --device=/dev/dri --ipc=host --shm-size 16G --group-add video --cap-add=SYS_PTRACE --cap-add=SYS_ADMIN + steps: + - name: Container setup + run: | + if [ -f /.dockerenv ]; then + apt-get update && apt-get install -y sudo && apt-get install -y git + git config --global --add safe.directory '*' + else + echo 'Skipping. Current job is not running inside a container.' + fi + - name: Checkout + uses: actions/checkout@v4 + - name: Build and Test (cmake) + uses: ./.github/actions/build_cmake + with: + gpu: ON + rocm: ON + linux-arm64-SVE-cmake: + name: Linux arm64 SVE (cmake) + needs: linux-x86_64-cmake + runs-on: faiss-aws-r8g.large + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Build and Test (cmake) + uses: ./.github/actions/build_cmake + with: + opt_level: sve + env: + # Context: https://github.com/facebookresearch/faiss/wiki/Troubleshooting#surprising-faiss-openmp-and-openblas-interaction + OPENBLAS_NUM_THREADS: '1' + linux-x86_64-conda: + name: Linux x86_64 (conda) + needs: linux-x86_64-cmake + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + - name: Build and Package (conda) + uses: ./.github/actions/build_conda + windows-x86_64-conda: + name: Windows x86_64 (conda) + needs: linux-x86_64-cmake + runs-on: windows-2019 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + - name: Build and Package (conda) + uses: ./.github/actions/build_conda + linux-arm64-conda: + name: Linux arm64 (conda) + needs: linux-x86_64-cmake + runs-on: 2-core-ubuntu-arm + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + - name: Build and Package (conda) + uses: ./.github/actions/build_conda diff --git a/.github/workflows/build-release.yml b/.github/workflows/build-release.yml new file mode 100644 index 0000000000..8343801037 --- /dev/null +++ b/.github/workflows/build-release.yml @@ -0,0 +1,142 @@ +name: Build Release +on: + workflow_call: +env: + OMP_NUM_THREADS: '10' + MKL_THREADING_LAYER: GNU +jobs: + linux-x86_64-packages: + name: Linux x86_64 packages + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + - name: Build and Package (conda) + uses: ./.github/actions/build_conda + env: + ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} + with: + label: main + linux-x86_64-GPU-packages-CUDA-11-4-4: + name: Linux x86_64 GPU packages (CUDA 11.4.4) + runs-on: 4-core-ubuntu-gpu-t4 + env: + CUDA_ARCHS: "60-real;61-real;62-real;70-real;72-real;75-real;80;86-real" + FAISS_FLATTEN_CONDA_INCLUDES: "1" + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + - name: Build and Package (conda) + uses: ./.github/actions/build_conda + env: + ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} + with: + label: main + cuda: "11.4.4" + linux-x86_64-GPU-CUVS-packages-CUDA11-8-0: + name: Linux x86_64 GPU w/ cuVS packages (CUDA 11.8.0) + runs-on: 4-core-ubuntu-gpu-t4 + env: + CUDA_ARCHS: "70-real;72-real;75-real;80;86-real" + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + - name: Build and Package (conda) + uses: ./.github/actions/build_conda + env: + ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} + with: + label: main + cuvs: "ON" + cuda: "11.8.0" + linux-x86_64-GPU-packages-CUDA-12-1-1: + name: Linux x86_64 GPU packages (CUDA 12.1.1) + runs-on: 4-core-ubuntu-gpu-t4 + env: + CUDA_ARCHS: "70-real;72-real;75-real;80;86-real" + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + - name: Build and Package (conda) + uses: ./.github/actions/build_conda + env: + ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} + with: + label: main + cuda: "12.1.1" + linux-x86_64-GPU-CUVS-packages-CUDA12-4-0: + name: Linux x86_64 GPU w/ cuVS packages (CUDA 12.4.0) + runs-on: 4-core-ubuntu-gpu-t4 + env: + CUDA_ARCHS: "70-real;72-real;75-real;80;86-real" + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + - name: Build and Package (conda) + uses: ./.github/actions/build_conda + env: + ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} + with: + label: main + cuvs: "ON" + cuda: "12.4.0" + windows-x86_64-packages: + name: Windows x86_64 packages + runs-on: windows-2019 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + - name: Build and Package (conda) + uses: ./.github/actions/build_conda + env: + ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} + with: + label: main + osx-arm64-packages: + name: OSX arm64 packages + runs-on: macos-14 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + - name: Build and Package (conda) + uses: ./.github/actions/build_conda + env: + ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} + with: + label: main + linux-arm64-packages: + name: Linux arm64 packages + runs-on: 2-core-ubuntu-arm + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + - name: Build and Package (conda) + uses: ./.github/actions/build_conda + env: + ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} + with: + label: main From e34ec93bed4aa58afedc97776ddb4be75d5f0e30 Mon Sep 17 00:00:00 2001 From: Junjie Qi Date: Thu, 5 Dec 2024 17:06:56 -0800 Subject: [PATCH 16/20] remove name --- .github/workflows/build-pull-request.yml | 1 - .github/workflows/build-release.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/.github/workflows/build-pull-request.yml b/.github/workflows/build-pull-request.yml index e5f3e6614d..d94abba081 100644 --- a/.github/workflows/build-pull-request.yml +++ b/.github/workflows/build-pull-request.yml @@ -1,4 +1,3 @@ -name: Build Pull Request on: workflow_call: env: diff --git a/.github/workflows/build-release.yml b/.github/workflows/build-release.yml index 8343801037..b859a4f287 100644 --- a/.github/workflows/build-release.yml +++ b/.github/workflows/build-release.yml @@ -1,4 +1,3 @@ -name: Build Release on: workflow_call: env: From e0ce4339154b1d5451c6217cffa15e8d8a62e463 Mon Sep 17 00:00:00 2001 From: Michael Norris Date: Thu, 5 Dec 2024 13:43:28 -0800 Subject: [PATCH 17/20] Add more unit testing for IndexHNSW [1/n] (#4054) Summary: Pull Request resolved: https://github.com/facebookresearch/faiss/pull/4054 Part 1 of more HNSW unit tests Reviewed By: junjieqi Differential Revision: D66690398 fbshipit-source-id: 29b35f1c0a919c168fd4cb770552f41eaed5b6b6 --- tests/test_graph_based.py | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/tests/test_graph_based.py b/tests/test_graph_based.py index 1f840e6cac..6fb60b3598 100644 --- a/tests/test_graph_based.py +++ b/tests/test_graph_based.py @@ -73,6 +73,18 @@ def test_hnsw_unbounded_queue(self): self.io_and_retest(index, Dhnsw, Ihnsw) + def test_hnsw_no_init_level0(self): + d = self.xq.shape[1] + + index = faiss.IndexHNSWFlat(d, 16) + index.init_level0 = False + index.add(self.xb) + Dhnsw, Ihnsw = index.search(self.xq, 1) + + self.assertGreaterEqual((self.Iref == Ihnsw).sum(), 25) + + self.io_and_retest(index, Dhnsw, Ihnsw) + def io_and_retest(self, index, Dhnsw, Ihnsw): index2 = faiss.deserialize_index(faiss.serialize_index(index)) Dhnsw2, Ihnsw2 = index2.search(self.xq, 1) @@ -175,7 +187,7 @@ def test_abs_inner_product(self): xb = self.xb - self.xb.mean(axis=0) # need to be centered to give interesting directions xq = self.xq - self.xq.mean(axis=0) Dref, Iref = faiss.knn(xq, xb, 10, faiss.METRIC_ABS_INNER_PRODUCT) - + index = faiss.IndexHNSWFlat(d, 32, faiss.METRIC_ABS_INNER_PRODUCT) index.add(xb) Dnew, Inew = index.search(xq, 10) @@ -183,8 +195,23 @@ def test_abs_inner_product(self): inter = faiss.eval_intersection(Iref, Inew) # 4769 vs. 500*10 self.assertGreater(inter, Iref.size * 0.9) - - + + def test_hnsw_reset(self): + d = self.xb.shape[1] + index_flat = faiss.IndexFlat(d) + index_flat.add(self.xb) + self.assertEqual(index_flat.ntotal, self.xb.shape[0]) + index_hnsw = faiss.IndexHNSW(index_flat) + index_hnsw.add(self.xb) + # * 2 because we add to storage twice. This is just for testing + # that storage gets cleared correctly. + self.assertEqual(index_hnsw.ntotal, self.xb.shape[0] * 2) + + index_hnsw.reset() + + self.assertEqual(index_flat.ntotal, 0) + self.assertEqual(index_hnsw.ntotal, 0) + class Issue3684(unittest.TestCase): def test_issue3684(self): From 92d6e3cb2d4e31aba6d3c34fc5e8c5edcd6d3a02 Mon Sep 17 00:00:00 2001 From: Mengdi Lin Date: Thu, 5 Dec 2024 14:21:38 -0800 Subject: [PATCH 18/20] Improve naming due to codemod (#4064) Summary: Pull Request resolved: https://github.com/facebookresearch/faiss/pull/4064 Apply matthijs' comments on better naming in D52582914 Reviewed By: junjieqi Differential Revision: D66822957 fbshipit-source-id: d899dbe89ebd716a18901c734c65dac428856954 --- faiss/IndexFastScan.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/faiss/IndexFastScan.cpp b/faiss/IndexFastScan.cpp index 8d10ecc9d9..b18d15bc17 100644 --- a/faiss/IndexFastScan.cpp +++ b/faiss/IndexFastScan.cpp @@ -33,22 +33,22 @@ inline size_t roundup(size_t a, size_t b) { void IndexFastScan::init_fastscan( int d, - size_t M_2, - size_t nbits_2, + size_t M_init, + size_t nbits_init, MetricType metric, int bbs) { - FAISS_THROW_IF_NOT(nbits_2 == 4); + FAISS_THROW_IF_NOT(nbits_init == 4); FAISS_THROW_IF_NOT(bbs % 32 == 0); this->d = d; - this->M = M_2; - this->nbits = nbits_2; + this->M = M_init; + this->nbits = nbits_init; this->metric_type = metric; this->bbs = bbs; - ksub = (1 << nbits_2); + ksub = (1 << nbits_init); - code_size = (M_2 * nbits_2 + 7) / 8; + code_size = (M_init * nbits_init + 7) / 8; ntotal = ntotal2 = 0; - M2 = roundup(M_2, 2); + M2 = roundup(M_init, 2); is_trained = false; } From 5f7774ee31caaabbc116471621f36611cac6d27c Mon Sep 17 00:00:00 2001 From: Michael Norris Date: Thu, 5 Dec 2024 15:14:14 -0800 Subject: [PATCH 19/20] Add more unit testing for IndexHNSW [2/n] (#4056) Summary: Pull Request resolved: https://github.com/facebookresearch/faiss/pull/4056 Part 2 of more HNSW unit tests Added comments indicating some currently unused code. Reviewed By: junjieqi Differential Revision: D66782376 fbshipit-source-id: 5a7c210d516424e58054f36f0a74249859e48a8f --- faiss/IndexHNSW.cpp | 11 +++++++++++ tests/test_graph_based.py | 20 ++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/faiss/IndexHNSW.cpp b/faiss/IndexHNSW.cpp index 6a40196f00..4cc58d211c 100644 --- a/faiss/IndexHNSW.cpp +++ b/faiss/IndexHNSW.cpp @@ -351,6 +351,17 @@ void IndexHNSW::reconstruct(idx_t key, float* recons) const { storage->reconstruct(key, recons); } +/************************************************************** + * This section of functions were used during the development of HNSW support. + * They may be useful in the future but are dormant for now, and thus are not + * unit tested at the moment. + * shrink_level_0_neighbors + * search_level_0 + * init_level_0_from_knngraph + * init_level_0_from_entry_points + * reorder_links + * link_singletons + **************************************************************/ void IndexHNSW::shrink_level_0_neighbors(int new_size) { #pragma omp parallel { diff --git a/tests/test_graph_based.py b/tests/test_graph_based.py index 6fb60b3598..a953617a8c 100644 --- a/tests/test_graph_based.py +++ b/tests/test_graph_based.py @@ -81,6 +81,8 @@ def test_hnsw_no_init_level0(self): index.add(self.xb) Dhnsw, Ihnsw = index.search(self.xq, 1) + # This is expected to be smaller because we are not initializing + # vectors into level 0. self.assertGreaterEqual((self.Iref == Ihnsw).sum(), 25) self.io_and_retest(index, Dhnsw, Ihnsw) @@ -113,6 +115,24 @@ def test_hnsw_2level(self): self.io_and_retest(index, Dhnsw, Ihnsw) + def test_hnsw_2level_mixed_search(self): + d = self.xq.shape[1] + + quant = faiss.IndexFlatL2(d) + + storage = faiss.IndexIVFPQ(quant, d, 32, 8, 8) + storage.make_direct_map() + index = faiss.IndexHNSW2Level(quant, 32, 8, 8) + index.storage = storage + index.train(self.xb) + index.add(self.xb) + Dhnsw, Ihnsw = index.search(self.xq, 1) + + # It is expected that the mixed search will perform worse. + self.assertGreaterEqual((self.Iref == Ihnsw).sum(), 200) + + self.io_and_retest(index, Dhnsw, Ihnsw) + def test_add_0_vecs(self): index = faiss.IndexHNSWFlat(10, 16) zero_vecs = np.zeros((0, 10), dtype='float32') From 70a385731dd1d21bec80b6b04df8627c8ace2a67 Mon Sep 17 00:00:00 2001 From: Mengdi Lin Date: Thu, 5 Dec 2024 15:32:14 -0800 Subject: [PATCH 20/20] Improve naming due to codemod (#4065) Summary: Pull Request resolved: https://github.com/facebookresearch/faiss/pull/4065 improve neighbors_2 naming from codemod Reviewed By: junjieqi Differential Revision: D66823395 fbshipit-source-id: 90af06e7f6d917581a25311944689f82f12c93e1 --- faiss/impl/HNSW.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/faiss/impl/HNSW.cpp b/faiss/impl/HNSW.cpp index 4f1ac7c3ba..642bf7c532 100644 --- a/faiss/impl/HNSW.cpp +++ b/faiss/impl/HNSW.cpp @@ -493,17 +493,17 @@ void HNSW::add_links_starting_from( ::faiss::shrink_neighbor_list(ptdis, link_targets, M, keep_max_size_level0); - std::vector neighbors_2; - neighbors_2.reserve(link_targets.size()); + std::vector neighbors_to_add; + neighbors_to_add.reserve(link_targets.size()); while (!link_targets.empty()) { storage_idx_t other_id = link_targets.top().id; add_link(*this, ptdis, pt_id, other_id, level, keep_max_size_level0); - neighbors_2.push_back(other_id); + neighbors_to_add.push_back(other_id); link_targets.pop(); } omp_unset_lock(&locks[pt_id]); - for (storage_idx_t other_id : neighbors_2) { + for (storage_idx_t other_id : neighbors_to_add) { omp_set_lock(&locks[other_id]); add_link(*this, ptdis, other_id, pt_id, level, keep_max_size_level0); omp_unset_lock(&locks[other_id]);