FAISS test TTI1M return r1@100=0

[SubjectNoi]

Summary

I try to do some benchmark on TTI1M dataset, but I get all recall1@100 to be 0

Platform

OS:

Faiss version: conda latest

Installed from: conda

Faiss compilation options:

Running on:

• CPU
• GPU

Interface:

• C++
• Python

Reproduction instructions

This file can reproduce:

import os                                                                                                                                                                       
import time
import numpy as np
import pdb 

import faiss
from faiss.contrib import inspect_tools
from datasets import load_tti1M, evaluate 

xb, xq, xt, gt = load_tti1M()
nq, d = xq.shape
res = faiss.StandardGpuResources()

index = faiss.index_factory(d, "IVF4096,PQ8", faiss.METRIC_INNER_PRODUCT)
co = faiss.GpuClonerOptions()
co.useFloat16 = True

index = faiss.index_cpu_to_gpu(res, 0, index, co) 
print("train")
index.train(xb)
print("add vectors to index")
index.add(xb)
print("benchmark")

for lnprobe in range(10):
    nprobe = 1 << lnprobe
    index.setNumProbes(nprobe)
    t, r = evaluate(index, xq, gt, 100)

    print(r[100])

The dataset.py:

from __future__ import print_function
import sys 
import time
import numpy as np


path = "/path/to/TTI1M/"

def read_fbin(filename, start_idx=0, chunk_size=None):
    """ Read *.fbin file that contains float32 vectors
    Args:
        :param filename (str): path to *.fbin file
        :param start_idx (int): start reading vectors from this index
        :param chunk_size (int): number of vectors to read. 
                                 If None, read all vectors
    Returns:
        Array of float32 vectors (numpy.ndarray)
    """
    with open(filename, "rb") as f:
        nvecs, dim = np.fromfile(f, count=2, dtype=np.int32)
        nvecs = (nvecs - start_idx) if chunk_size is None else chunk_size
        arr = np.fromfile(f, count=nvecs * dim, dtype=np.float32, 
                          offset=start_idx * 4 * dim)
    return arr.reshape(nvecs, dim)
 
 
def read_ibin(filename, start_idx=0, chunk_size=None):
    """ Read *.ibin file that contains int32 vectors
    Args:
        :param filename (str): path to *.ibin file
        :param start_idx (int): start reading vectors from this index
        :param chunk_size (int): number of vectors to read.
                                 If None, read all vectors
    Returns:
        Array of int32 vectors (numpy.ndarray)
    """
    with open(filename, "rb") as f:
        nvecs, dim = np.fromfile(f, count=2, dtype=np.int32)
        nvecs = (nvecs - start_idx) if chunk_size is None else chunk_size
        arr = np.fromfile(f, count=nvecs * dim, dtype=np.int32, 
                          offset=start_idx * 4 * dim)
    return arr.reshape(nvecs, dim)

def load_tti1M():
    print("Loading tti1M...", end='', file=sys.stderr)
    xt = []
    xb = read_fbin(path + "base.1M.fbin")
    xq = read_fbin(path + "query.public.100K.fbin", 0, 10000)
    gt = read_ibin(path + "groundtruth.public.100K.ibin", 0, 10000)
    print("done", file=sys.stderr)
    return xb, xq, xt, gt

def evaluate(index, xq, gt, k):
    nq = xq.shape[0]
    t0 = time.time()
    D, I = index.search(xq, k)  # noqa: E741
    t1 = time.time()
    r100_1000 = 0.0
    
    for q in range(nq):
        gt100 = gt[q][0:100]
        cnt = 0
        for g in I[q][0:1000]:
            if g in gt100:
                cnt += 1
        r100_1000 += (1.0 * cnt) / (100.0)
    
    recalls = {}
    i = 1
    while i <= k:
        recalls[i] = (I[:, :i] == gt[:, :1]).sum() / float(nq)
        i *= 10
    recalls[1000] = r100_1000 / (1.0 * nq)
    return (t1 - t0) * 1000.0, recalls

[mdouze]

What is TTI ?
Please check your evaluation code with a brute force index first.