diff --git a/mteb/models/overview.py b/mteb/models/overview.py index e68a4f65e4..340989baaf 100644 --- a/mteb/models/overview.py +++ b/mteb/models/overview.py @@ -44,6 +44,7 @@ vista_models, vlm2vec_models, voyage_models, + voyage_v, ) logger = logging.getLogger(__name__) @@ -81,6 +82,7 @@ siglip_models, vista_models, voyage_models, + voyage_v, vlm2vec_models, repllama_models, promptriever_models, diff --git a/mteb/models/voyage_v.py b/mteb/models/voyage_v.py new file mode 100644 index 0000000000..c1787ea10f --- /dev/null +++ b/mteb/models/voyage_v.py @@ -0,0 +1,179 @@ +from __future__ import annotations + +import os +from functools import partial +from typing import Any + +import torch +from PIL import Image +from torch.utils.data import DataLoader +from torchvision import transforms +from tqdm import tqdm + +import mteb +from mteb.model_meta import ModelMeta + +api_key = os.getenv("VOYAGE_API_KEY") +tensor_to_image = transforms.Compose([transforms.ToPILImage()]) + + +def voyage_v_loader(**kwargs): + try: + import voyageai + except ImportError: + raise ImportError("To use voyage models, please run `pip install -U voyageai`.") + + class VoyageMultiModalModelWrapper: + def __init__( + self, + model_name: str, + **kwargs: Any, + ): + self.model_name = model_name + self.vo = voyageai.Client() + + def get_text_embeddings( + self, texts: list[str], batch_size: int = 32, input_type=None + ): + all_text_embeddings = [] + + for i in tqdm(range(0, len(texts), batch_size)): + batch_texts = texts[i : i + batch_size] + batch_texts = [[text] for text in batch_texts] + all_text_embeddings += torch.tensor( + self.vo.multimodal_embed( + batch_texts, model=self.model_name, input_type=input_type + ).embeddings + ) + all_text_embeddings = torch.vstack(all_text_embeddings) + return all_text_embeddings + + def get_image_embeddings( + self, + images: list[Image.Image] | DataLoader, + batch_size: int = 32, + input_type=None, + ): + all_image_embeddings = [] + + if isinstance(images, DataLoader): + for index, batch in enumerate(tqdm(images)): + if index == 0: + assert len(batch) == batch_size + batch_images = [[tensor_to_image(image)] for image in batch] + all_image_embeddings += torch.tensor( + self.vo.multimodal_embed( + batch_images, model=self.model_name, input_type=input_type + ).embeddings + ) + else: + for i in tqdm(range(0, len(images), batch_size)): + batch_images = images[i : i + batch_size] + batch_images = [[image] for image in batch_images] + all_image_embeddings += torch.tensor( + self.vo.multimodal_embed( + batch_images, model=self.model_name, input_type=input_type + ).embeddings + ) + all_image_embeddings = torch.vstack(all_image_embeddings) + return all_image_embeddings + + def calculate_probs(self, text_embeddings, image_embeddings): + text_embeddings = text_embeddings / text_embeddings.norm( + dim=-1, keepdim=True + ) + image_embeddings = image_embeddings / image_embeddings.norm( + dim=-1, keepdim=True + ) + logits = torch.matmul(image_embeddings, text_embeddings.T) + probs = (logits * 100).softmax(dim=-1) + return probs + + def get_fused_embeddings( + self, + texts: list[str] = None, + images: list[Image.Image] | DataLoader = None, + batch_size: int = 32, + input_type=None, + ): + if texts is None and images is None: + raise ValueError("Either texts or images must be provided") + + text_embeddings = None + image_embeddings = None + + interleaved_embeddings = [] + if texts is not None and images is not None: + # print("encoding interleaved inputs") + if isinstance(images, DataLoader): + for index, batch in tqdm(enumerate(images)): + if index == 0: + assert len(batch) == batch_size + batch_images = [tensor_to_image(image) for image in batch] + batch_texts = texts[ + index * batch_size : (index + 1) * batch_size + ] + interleaved_inputs = [ + [text, image] + for image, text in zip(batch_images, batch_texts) + ] + interleaved_embeddings += torch.tensor( + self.vo.multimodal_embed( + interleaved_inputs, + model=self.model_name, + input_type=input_type, + ).embeddings + ) + else: + for i in tqdm(range(0, len(images), batch_size)): + batch_images = images[i : i + batch_size] + batch_texts = texts[i : i + batch_size] + interleaved_inputs = [ + [text, image] + for image, text in zip(batch_images, batch_texts) + ] + interleaved_embeddings += torch.tensor( + self.vo.multimodal_embed( + interleaved_inputs, + model=self.model_name, + input_type=input_type, + ).embeddings + ) + interleaved_embeddings = torch.vstack(interleaved_embeddings) + return interleaved_embeddings + + elif texts is not None: + # print("encoding texts only") + text_embeddings = self.get_text_embeddings(texts, batch_size) + + elif images is not None: + # print("encoding images only") + image_embeddings = self.get_image_embeddings(images, batch_size) + + if text_embeddings is not None: + return text_embeddings + elif image_embeddings is not None: + return image_embeddings + + return VoyageMultiModalModelWrapper(**kwargs) + + +cohere_mult_3 = ModelMeta( + loader=partial(voyage_v_loader, model_name="voyage-multimodal-3"), + name="voyage-multimodal-3", + languages=[], # Unknown + open_source=False, + revision="1", + release_date="2024-11-10", + n_parameters=None, + memory_usage=None, + max_tokens=None, + embed_dim=1024, + license=None, + similarity_fn_name="cosine", + framework=[], +) + +if __name__ == "__main__": + mdl = mteb.get_model(cohere_mult_3.name, cohere_mult_3.revision) + emb = mdl.encode(["Hello, world!"]) diff --git a/results-mieb/voyage-multimodal-3/1/CIFAR100Clustering.json b/results-mieb/voyage-multimodal-3/1/CIFAR100Clustering.json new file mode 100644 index 0000000000..221465417f --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/CIFAR100Clustering.json @@ -0,0 +1,22 @@ +{ + "dataset_revision": "aadb3af77e9048adbea6b47c21a81e47dd092ae5", + "evaluation_time": 231.98164081573486, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "ari": 0.46531392836213886, + "cluster_accuracy": 0.6042, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.7515067668506068, + "nmi": 0.7515067668506068, + "v_measure": 0.7515067668506068 + } + ] + }, + "task_name": "CIFAR100Clustering" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/CIFAR10Clustering.json b/results-mieb/voyage-multimodal-3/1/CIFAR10Clustering.json new file mode 100644 index 0000000000..5459074f45 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/CIFAR10Clustering.json @@ -0,0 +1,22 @@ +{ + "dataset_revision": "0b2714987fa478483af9968de7c934580d0bb9a2", + "evaluation_time": 236.98327565193176, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "ari": 0.7986159784876974, + "cluster_accuracy": 0.8244, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8622309717386565, + "nmi": 0.8622309717386565, + "v_measure": 0.8622309717386566 + } + ] + }, + "task_name": "CIFAR10Clustering" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/CVBenchCount.json b/results-mieb/voyage-multimodal-3/1/CVBenchCount.json new file mode 100644 index 0000000000..ff443295ee --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/CVBenchCount.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "22409a927ab5cf68e3655023d51694587455fc99", + "evaluation_time": 444.4619266986847, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.2639593908629442, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.2639593908629442 + } + ] + }, + "task_name": "CVBenchCount" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/CVBenchDepth.json b/results-mieb/voyage-multimodal-3/1/CVBenchDepth.json new file mode 100644 index 0000000000..185e34679a --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/CVBenchDepth.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "22409a927ab5cf68e3655023d51694587455fc99", + "evaluation_time": 413.3635849952698, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.5316666666666666, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.5316666666666666 + } + ] + }, + "task_name": "CVBenchDepth" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/CVBenchDistance.json b/results-mieb/voyage-multimodal-3/1/CVBenchDistance.json new file mode 100644 index 0000000000..a950524030 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/CVBenchDistance.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "22409a927ab5cf68e3655023d51694587455fc99", + "evaluation_time": 402.95056080818176, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.475, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.475 + } + ] + }, + "task_name": "CVBenchDistance" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/CVBenchRelation.json b/results-mieb/voyage-multimodal-3/1/CVBenchRelation.json new file mode 100644 index 0000000000..9a75e5c540 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/CVBenchRelation.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "22409a927ab5cf68e3655023d51694587455fc99", + "evaluation_time": 201.59757685661316, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "accuracy": 0.5353846153846153, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.5353846153846153 + } + ] + }, + "task_name": "CVBenchRelation" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/ImageNet10Clustering.json b/results-mieb/voyage-multimodal-3/1/ImageNet10Clustering.json new file mode 100644 index 0000000000..750953aea8 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/ImageNet10Clustering.json @@ -0,0 +1,22 @@ +{ + "dataset_revision": "88f8a6d47c257895094c5ad81e67ba751771fc99", + "evaluation_time": 914.053968667984, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "ari": 0.9808613040028806, + "cluster_accuracy": 0.9913076923076923, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.9758070118084365, + "nmi": 0.9758070118084365, + "v_measure": 0.9758070118084365 + } + ] + }, + "task_name": "ImageNet10Clustering" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/ImageNetDog15Clustering.json b/results-mieb/voyage-multimodal-3/1/ImageNetDog15Clustering.json new file mode 100644 index 0000000000..96f915915f --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/ImageNetDog15Clustering.json @@ -0,0 +1,22 @@ +{ + "dataset_revision": "bfb6ad3b2109d26c9daddf14f98d315daa35ee72", + "evaluation_time": 78.63793706893921, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "ari": 0.8119121646758497, + "cluster_accuracy": 0.820631970260223, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8381839891894807, + "nmi": 0.8381839891894807, + "v_measure": 0.8381839891894807 + } + ] + }, + "task_name": "ImageNetDog15Clustering" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/STS12VisualSTS.json b/results-mieb/voyage-multimodal-3/1/STS12VisualSTS.json new file mode 100644 index 0000000000..0d5231319d --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/STS12VisualSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "820c25edfba736f3789201b2476208cc62c2ccb9", + "evaluation_time": 381.2039792537689, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cosine_pearson": 0.7530408956307458, + "cosine_spearman": 0.7161988759864429, + "euclidean_pearson": 0.7141309522927283, + "euclidean_spearman": 0.7162724788414607, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.7161988759864429, + "manhattan_pearson": 0.7140835255725306, + "manhattan_spearman": 0.7154340341352178, + "pearson": 0.7530408956307458, + "spearman": 0.7161988759864429 + } + ] + }, + "task_name": "STS12VisualSTS" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/STS13VisualSTS.json b/results-mieb/voyage-multimodal-3/1/STS13VisualSTS.json new file mode 100644 index 0000000000..f422f81ad5 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/STS13VisualSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "561ee9ca47ff3e4a657283c59416deca8dc169f2", + "evaluation_time": 178.80085134506226, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cosine_pearson": 0.8041898098977802, + "cosine_spearman": 0.8160271504089486, + "euclidean_pearson": 0.8138453622908126, + "euclidean_spearman": 0.8160367132733977, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8160271504089486, + "manhattan_pearson": 0.8141249912528841, + "manhattan_spearman": 0.816315080890506, + "pearson": 0.8041898098977802, + "spearman": 0.8160271504089486 + } + ] + }, + "task_name": "STS13VisualSTS" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/STS14VisualSTS.json b/results-mieb/voyage-multimodal-3/1/STS14VisualSTS.json new file mode 100644 index 0000000000..54abe0a859 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/STS14VisualSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "824e95e45471024a684b901e0645579ffd9ca288", + "evaluation_time": 477.0949454307556, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cosine_pearson": 0.8126428329540769, + "cosine_spearman": 0.7798398073449294, + "euclidean_pearson": 0.8019773377625052, + "euclidean_spearman": 0.779773229647281, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.7798398073449294, + "manhattan_pearson": 0.8019854029161337, + "manhattan_spearman": 0.7797635347309039, + "pearson": 0.8126428329540769, + "spearman": 0.7798398073449294 + } + ] + }, + "task_name": "STS14VisualSTS" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/STS15VisualSTS.json b/results-mieb/voyage-multimodal-3/1/STS15VisualSTS.json new file mode 100644 index 0000000000..d2a32dd340 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/STS15VisualSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "1f8d08d9b9daac7118dfdefeb94b0aac4baf2e5f", + "evaluation_time": 385.04992389678955, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cosine_pearson": 0.8596747384607906, + "cosine_spearman": 0.8684963727234637, + "euclidean_pearson": 0.8648294270010821, + "euclidean_spearman": 0.8684429467437428, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8684963727234637, + "manhattan_pearson": 0.8645923546880342, + "manhattan_spearman": 0.8682092035635413, + "pearson": 0.8596747384607906, + "spearman": 0.8684963727234637 + } + ] + }, + "task_name": "STS15VisualSTS" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/STS16VisualSTS.json b/results-mieb/voyage-multimodal-3/1/STS16VisualSTS.json new file mode 100644 index 0000000000..653d939404 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/STS16VisualSTS.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "fc354f19598af93f32c0af1b94046ffdeaacde15", + "evaluation_time": 150.6804370880127, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cosine_pearson": 0.8169368709648801, + "cosine_spearman": 0.8261532729608647, + "euclidean_pearson": 0.8216160911211441, + "euclidean_spearman": 0.8260077161971415, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8261532729608647, + "manhattan_pearson": 0.8212353007123708, + "manhattan_spearman": 0.8258613312454957, + "pearson": 0.8169368709648801, + "spearman": 0.8261532729608647 + } + ] + }, + "task_name": "STS16VisualSTS" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/STS17MultilingualVisualSTS.json b/results-mieb/voyage-multimodal-3/1/STS17MultilingualVisualSTS.json new file mode 100644 index 0000000000..8bde9d8217 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/STS17MultilingualVisualSTS.json @@ -0,0 +1,183 @@ +{ + "dataset_revision": "2e31b4b459551a51e1ab54fd7266b40f3fe510d4", + "evaluation_time": 693.5000240802765, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cosine_pearson": 0.6295478701037069, + "cosine_spearman": 0.6279910477771019, + "euclidean_pearson": 0.6276223489798609, + "euclidean_spearman": 0.6280662784030709, + "hf_subset": "ko-ko", + "languages": [ + "kor-Hang" + ], + "main_score": 0.6279910477771019, + "manhattan_pearson": 0.6272252575072031, + "manhattan_spearman": 0.6275841741159163, + "pearson": 0.6295478701037069, + "spearman": 0.6279910477771019 + }, + { + "cosine_pearson": 0.6640588532698395, + "cosine_spearman": 0.6575414606953666, + "euclidean_pearson": 0.6455970584249845, + "euclidean_spearman": 0.6568954839487309, + "hf_subset": "ar-ar", + "languages": [ + "ara-Arab" + ], + "main_score": 0.6575414606953666, + "manhattan_pearson": 0.6456300855443649, + "manhattan_spearman": 0.6581063058387047, + "pearson": 0.6640588532698395, + "spearman": 0.6575414606953666 + }, + { + "cosine_pearson": 0.3785792513239742, + "cosine_spearman": 0.3440485632827937, + "euclidean_pearson": 0.3818330414668889, + "euclidean_spearman": 0.3425467167114543, + "hf_subset": "en-ar", + "languages": [ + "eng-Latn", + "ara-Arab" + ], + "main_score": 0.3440485632827937, + "manhattan_pearson": 0.37280091876138727, + "manhattan_spearman": 0.33497021404416516, + "pearson": 0.3785792513239742, + "spearman": 0.3440485632827937 + }, + { + "cosine_pearson": 0.797388779176213, + "cosine_spearman": 0.8041814174327988, + "euclidean_pearson": 0.8018372869118563, + "euclidean_spearman": 0.8046465372595982, + "hf_subset": "en-de", + "languages": [ + "eng-Latn", + "deu-Latn" + ], + "main_score": 0.8041814174327988, + "manhattan_pearson": 0.8036367691934629, + "manhattan_spearman": 0.806153756136359, + "pearson": 0.797388779176213, + "spearman": 0.8041814174327988 + }, + { + "cosine_pearson": 0.889324389401427, + "cosine_spearman": 0.8968348245174512, + "euclidean_pearson": 0.8988351349236602, + "euclidean_spearman": 0.8966841410694303, + "hf_subset": "en-en", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8968348245174512, + "manhattan_pearson": 0.8987261929400799, + "manhattan_spearman": 0.8980541303774577, + "pearson": 0.889324389401427, + "spearman": 0.8968348245174512 + }, + { + "cosine_pearson": 0.46493619455982876, + "cosine_spearman": 0.4498243279387666, + "euclidean_pearson": 0.46435399844913355, + "euclidean_spearman": 0.451091543774284, + "hf_subset": "en-tr", + "languages": [ + "eng-Latn", + "tur-Latn" + ], + "main_score": 0.4498243279387666, + "manhattan_pearson": 0.4687785223213941, + "manhattan_spearman": 0.4568005510381185, + "pearson": 0.46493619455982876, + "spearman": 0.4498243279387666 + }, + { + "cosine_pearson": 0.7413496252477598, + "cosine_spearman": 0.7471866607172193, + "euclidean_pearson": 0.7483417196429633, + "euclidean_spearman": 0.7466169330700719, + "hf_subset": "es-en", + "languages": [ + "spa-Latn", + "eng-Latn" + ], + "main_score": 0.7471866607172193, + "manhattan_pearson": 0.7497169457615618, + "manhattan_spearman": 0.749720065032133, + "pearson": 0.7413496252477598, + "spearman": 0.7471866607172193 + }, + { + "cosine_pearson": 0.8401635646321107, + "cosine_spearman": 0.8370332502012957, + "euclidean_pearson": 0.850259304444977, + "euclidean_spearman": 0.8371101547419473, + "hf_subset": "es-es", + "languages": [ + "spa-Latn" + ], + "main_score": 0.8370332502012957, + "manhattan_pearson": 0.8511002525095128, + "manhattan_spearman": 0.8384332973638572, + "pearson": 0.8401635646321107, + "spearman": 0.8370332502012957 + }, + { + "cosine_pearson": 0.7447131067366383, + "cosine_spearman": 0.750669573557799, + "euclidean_pearson": 0.7452718819569142, + "euclidean_spearman": 0.7508660001953976, + "hf_subset": "fr-en", + "languages": [ + "fra-Latn", + "eng-Latn" + ], + "main_score": 0.750669573557799, + "manhattan_pearson": 0.7480003355590945, + "manhattan_spearman": 0.7524451012272094, + "pearson": 0.7447131067366383, + "spearman": 0.750669573557799 + }, + { + "cosine_pearson": 0.7685526370938169, + "cosine_spearman": 0.7776349916156341, + "euclidean_pearson": 0.7692583708827287, + "euclidean_spearman": 0.7780436051494257, + "hf_subset": "it-en", + "languages": [ + "ita-Latn", + "eng-Latn" + ], + "main_score": 0.7776349916156341, + "manhattan_pearson": 0.7680758399408012, + "manhattan_spearman": 0.776599427307041, + "pearson": 0.7685526370938169, + "spearman": 0.7776349916156341 + }, + { + "cosine_pearson": 0.7848622824203596, + "cosine_spearman": 0.7824649342798778, + "euclidean_pearson": 0.7849080671012388, + "euclidean_spearman": 0.7823646067800476, + "hf_subset": "nl-en", + "languages": [ + "nld-Latn", + "eng-Latn" + ], + "main_score": 0.7824649342798778, + "manhattan_pearson": 0.7863829683396525, + "manhattan_spearman": 0.7856565788434446, + "pearson": 0.7848622824203596, + "spearman": 0.7824649342798778 + } + ] + }, + "task_name": "STS17MultilingualVisualSTS" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/STSBenchmarkMultilingualVisualSTS.json b/results-mieb/voyage-multimodal-3/1/STSBenchmarkMultilingualVisualSTS.json new file mode 100644 index 0000000000..2a7ee75282 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/STSBenchmarkMultilingualVisualSTS.json @@ -0,0 +1,313 @@ +{ + "dataset_revision": "9f1ab21f17f497974996ab74b3ff911165a7dbf9", + "evaluation_time": 5075.43874335289, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "dev": [ + { + "cosine_pearson": 0.8563893474191902, + "cosine_spearman": 0.858058860838852, + "euclidean_pearson": 0.8602263464877766, + "euclidean_spearman": 0.8579783803562652, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ], + "main_score": 0.858058860838852, + "manhattan_pearson": 0.8598337346826477, + "manhattan_spearman": 0.8575681612119586, + "pearson": 0.8563893474191902, + "spearman": 0.858058860838852 + }, + { + "cosine_pearson": 0.789854359195163, + "cosine_spearman": 0.7899033365671463, + "euclidean_pearson": 0.7916778007557717, + "euclidean_spearman": 0.7898486229980857, + "hf_subset": "de", + "languages": [ + "deu-Latn" + ], + "main_score": 0.7899033365671463, + "manhattan_pearson": 0.7910013245869385, + "manhattan_spearman": 0.7894274945306929, + "pearson": 0.789854359195163, + "spearman": 0.7899033365671463 + }, + { + "cosine_pearson": 0.796760464557315, + "cosine_spearman": 0.8013993811687563, + "euclidean_pearson": 0.8032555282023144, + "euclidean_spearman": 0.8015704115360485, + "hf_subset": "es", + "languages": [ + "spa-Latn" + ], + "main_score": 0.8013993811687563, + "manhattan_pearson": 0.8020719661864791, + "manhattan_spearman": 0.8003944783391376, + "pearson": 0.796760464557315, + "spearman": 0.8013993811687563 + }, + { + "cosine_pearson": 0.7985932474785588, + "cosine_spearman": 0.8021648661142508, + "euclidean_pearson": 0.8031042634558174, + "euclidean_spearman": 0.8021820102550952, + "hf_subset": "fr", + "languages": [ + "fra-Latn" + ], + "main_score": 0.8021648661142508, + "manhattan_pearson": 0.8029724285823261, + "manhattan_spearman": 0.8019900652727078, + "pearson": 0.7985932474785588, + "spearman": 0.8021648661142508 + }, + { + "cosine_pearson": 0.7869557821233382, + "cosine_spearman": 0.7890330736912589, + "euclidean_pearson": 0.7906461233218647, + "euclidean_spearman": 0.7890663388388003, + "hf_subset": "it", + "languages": [ + "ita-Latn" + ], + "main_score": 0.7890330736912589, + "manhattan_pearson": 0.7903840939372646, + "manhattan_spearman": 0.7888588814000735, + "pearson": 0.7869557821233382, + "spearman": 0.7890330736912589 + }, + { + "cosine_pearson": 0.7791772215249965, + "cosine_spearman": 0.781520955767216, + "euclidean_pearson": 0.7801465349415607, + "euclidean_spearman": 0.7815041664138321, + "hf_subset": "nl", + "languages": [ + "nld-Latn" + ], + "main_score": 0.781520955767216, + "manhattan_pearson": 0.7789332520700933, + "manhattan_spearman": 0.7801130952911972, + "pearson": 0.7791772215249965, + "spearman": 0.781520955767216 + }, + { + "cosine_pearson": 0.7448849902931678, + "cosine_spearman": 0.7448043158631151, + "euclidean_pearson": 0.7405744069410145, + "euclidean_spearman": 0.7447120862522622, + "hf_subset": "pl", + "languages": [ + "pol-Latn" + ], + "main_score": 0.7448043158631151, + "manhattan_pearson": 0.7406202109132378, + "manhattan_spearman": 0.7445824918429544, + "pearson": 0.7448849902931678, + "spearman": 0.7448043158631151 + }, + { + "cosine_pearson": 0.779680602143197, + "cosine_spearman": 0.7826858098266277, + "euclidean_pearson": 0.7852151538285848, + "euclidean_spearman": 0.7826847689006386, + "hf_subset": "pt", + "languages": [ + "por-Latn" + ], + "main_score": 0.7826858098266277, + "manhattan_pearson": 0.7839441476775062, + "manhattan_spearman": 0.7813235864617706, + "pearson": 0.779680602143197, + "spearman": 0.7826858098266277 + }, + { + "cosine_pearson": 0.7884272004913433, + "cosine_spearman": 0.7892230784770122, + "euclidean_pearson": 0.7805909078732953, + "euclidean_spearman": 0.7893512299219531, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7892230784770122, + "manhattan_pearson": 0.7794459614806362, + "manhattan_spearman": 0.7878490798979422, + "pearson": 0.7884272004913433, + "spearman": 0.7892230784770122 + }, + { + "cosine_pearson": 0.7654432441204759, + "cosine_spearman": 0.7647755317149096, + "euclidean_pearson": 0.7593503194362672, + "euclidean_spearman": 0.7647270138720049, + "hf_subset": "zh", + "languages": [ + "cmn-Hans" + ], + "main_score": 0.7647755317149096, + "manhattan_pearson": 0.7589361597204412, + "manhattan_spearman": 0.764541990552287, + "pearson": 0.7654432441204759, + "spearman": 0.7647755317149096 + } + ], + "test": [ + { + "cosine_pearson": 0.823492051147191, + "cosine_spearman": 0.8255498065512306, + "euclidean_pearson": 0.826698523592022, + "euclidean_spearman": 0.8254643124686717, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8255498065512306, + "manhattan_pearson": 0.8263307385343645, + "manhattan_spearman": 0.8247261005744897, + "pearson": 0.823492051147191, + "spearman": 0.8255498065512306 + }, + { + "cosine_pearson": 0.7492177185806835, + "cosine_spearman": 0.7413351070642012, + "euclidean_pearson": 0.7439940554472478, + "euclidean_spearman": 0.7414010247347829, + "hf_subset": "de", + "languages": [ + "deu-Latn" + ], + "main_score": 0.7413351070642012, + "manhattan_pearson": 0.7436240717647824, + "manhattan_spearman": 0.7409998672149616, + "pearson": 0.7492177185806835, + "spearman": 0.7413351070642012 + }, + { + "cosine_pearson": 0.7607906213948373, + "cosine_spearman": 0.7599250153963639, + "euclidean_pearson": 0.7564171074765642, + "euclidean_spearman": 0.7598909226716342, + "hf_subset": "es", + "languages": [ + "spa-Latn" + ], + "main_score": 0.7599250153963639, + "manhattan_pearson": 0.7558868227763365, + "manhattan_spearman": 0.7597894599923053, + "pearson": 0.7607906213948373, + "spearman": 0.7599250153963639 + }, + { + "cosine_pearson": 0.7520573823123218, + "cosine_spearman": 0.7442880594186052, + "euclidean_pearson": 0.7420168403959864, + "euclidean_spearman": 0.7442472350562676, + "hf_subset": "fr", + "languages": [ + "fra-Latn" + ], + "main_score": 0.7442880594186052, + "manhattan_pearson": 0.7415021156781515, + "manhattan_spearman": 0.7434403372096718, + "pearson": 0.7520573823123218, + "spearman": 0.7442880594186052 + }, + { + "cosine_pearson": 0.7467690218563255, + "cosine_spearman": 0.7395905416647525, + "euclidean_pearson": 0.7357731015973681, + "euclidean_spearman": 0.7394434799461104, + "hf_subset": "it", + "languages": [ + "ita-Latn" + ], + "main_score": 0.7395905416647525, + "manhattan_pearson": 0.7350784821793559, + "manhattan_spearman": 0.7386311414238852, + "pearson": 0.7467690218563255, + "spearman": 0.7395905416647525 + }, + { + "cosine_pearson": 0.7202011364978524, + "cosine_spearman": 0.7134491712781116, + "euclidean_pearson": 0.7118458785635648, + "euclidean_spearman": 0.7134129390083591, + "hf_subset": "nl", + "languages": [ + "nld-Latn" + ], + "main_score": 0.7134491712781116, + "manhattan_pearson": 0.7109658343515404, + "manhattan_spearman": 0.7126032622007037, + "pearson": 0.7202011364978524, + "spearman": 0.7134491712781116 + }, + { + "cosine_pearson": 0.6985464798474408, + "cosine_spearman": 0.6883224080503167, + "euclidean_pearson": 0.6874216559416312, + "euclidean_spearman": 0.6881582413464393, + "hf_subset": "pl", + "languages": [ + "pol-Latn" + ], + "main_score": 0.6883224080503167, + "manhattan_pearson": 0.6871835930951957, + "manhattan_spearman": 0.6879319205502966, + "pearson": 0.6985464798474408, + "spearman": 0.6883224080503167 + }, + { + "cosine_pearson": 0.7362293011231412, + "cosine_spearman": 0.7348219968439652, + "euclidean_pearson": 0.7298573693202373, + "euclidean_spearman": 0.7346749511887691, + "hf_subset": "pt", + "languages": [ + "por-Latn" + ], + "main_score": 0.7348219968439652, + "manhattan_pearson": 0.7295003107001472, + "manhattan_spearman": 0.73423890877604, + "pearson": 0.7362293011231412, + "spearman": 0.7348219968439652 + }, + { + "cosine_pearson": 0.7368742675667894, + "cosine_spearman": 0.7267810144277265, + "euclidean_pearson": 0.7227644385927088, + "euclidean_spearman": 0.726788301200454, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7267810144277265, + "manhattan_pearson": 0.721898680516144, + "manhattan_spearman": 0.7256356052253394, + "pearson": 0.7368742675667894, + "spearman": 0.7267810144277265 + }, + { + "cosine_pearson": 0.7345816344328195, + "cosine_spearman": 0.7259656655754602, + "euclidean_pearson": 0.7169100386731838, + "euclidean_spearman": 0.7260215086810657, + "hf_subset": "zh", + "languages": [ + "cmn-Hans" + ], + "main_score": 0.7259656655754602, + "manhattan_pearson": 0.7164561544415515, + "manhattan_spearman": 0.7256700966168421, + "pearson": 0.7345816344328195, + "spearman": 0.7259656655754602 + } + ] + }, + "task_name": "STSBenchmarkMultilingualVisualSTS" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/TinyImageNetClustering.json b/results-mieb/voyage-multimodal-3/1/TinyImageNetClustering.json new file mode 100644 index 0000000000..0aa996e4ba --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/TinyImageNetClustering.json @@ -0,0 +1,22 @@ +{ + "dataset_revision": "5a77092c28e51558c5586e9c5eb71a7e17a5e43f", + "evaluation_time": 368.8909206390381, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "valid": [ + { + "ari": 0.31776897199787596, + "cluster_accuracy": 0.4675, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.6929420627528728, + "nmi": 0.6929420627528728, + "v_measure": 0.6929420627528728 + } + ] + }, + "task_name": "TinyImageNetClustering" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/XFlickr30kCoT2IRetrieval.json b/results-mieb/voyage-multimodal-3/1/XFlickr30kCoT2IRetrieval.json new file mode 100644 index 0000000000..8088fb6cce --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/XFlickr30kCoT2IRetrieval.json @@ -0,0 +1,1411 @@ +{ + "dataset_revision": "0af2c2eba58b27a71898787e286be04befdd7a20", + "evaluation_time": 3659.7448897361755, + "kg_co2_emissions": null, + "mteb_version": "1.14.15", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.6755, + "cv_recall_at_10": 0.929, + "cv_recall_at_100": 0.9865, + "cv_recall_at_1000": 0.9985, + "cv_recall_at_20": 0.9585, + "cv_recall_at_3": 0.8335, + "cv_recall_at_5": 0.887, + "hf_subset": "de", + "languages": [ + "deu-Latn" + ], + "main_score": 0.80482, + "map_at_1": 0.6755, + "map_at_10": 0.76463, + "map_at_100": 0.7676, + "map_at_1000": 0.76767, + "map_at_20": 0.76683, + "map_at_3": 0.7465, + "map_at_5": 0.75875, + "mrr_at_1": 0.6755, + "mrr_at_10": 0.764634126984127, + "mrr_at_100": 0.7675982824740856, + "mrr_at_1000": 0.767672981003628, + "mrr_at_20": 0.7668271411380156, + "mrr_at_3": 0.7464999999999994, + "mrr_at_5": 0.7587499999999999, + "nauc_cv_recall_at_1000_diff1": 0.8513849984438709, + "nauc_cv_recall_at_1000_max": 1.0, + "nauc_cv_recall_at_1000_std": 1.0, + "nauc_cv_recall_at_100_diff1": 0.7822561123214679, + "nauc_cv_recall_at_100_max": 0.9683577134557472, + "nauc_cv_recall_at_100_std": 0.8572120206107111, + "nauc_cv_recall_at_10_diff1": 0.6544594363567012, + "nauc_cv_recall_at_10_max": 0.7857274365145103, + "nauc_cv_recall_at_10_std": 0.30422403703265355, + "nauc_cv_recall_at_1_diff1": 0.7463322614632609, + "nauc_cv_recall_at_1_max": 0.5806226318361674, + "nauc_cv_recall_at_1_std": 0.10438451063582097, + "nauc_cv_recall_at_20_diff1": 0.6694002902365792, + "nauc_cv_recall_at_20_max": 0.8863802549132098, + "nauc_cv_recall_at_20_std": 0.5763220951031016, + "nauc_cv_recall_at_3_diff1": 0.6924928185360167, + "nauc_cv_recall_at_3_max": 0.6660863314034848, + "nauc_cv_recall_at_3_std": 0.15626155616629717, + "nauc_cv_recall_at_5_diff1": 0.6826023673900833, + "nauc_cv_recall_at_5_max": 0.7265547008045103, + "nauc_cv_recall_at_5_std": 0.21227234213903992, + "nauc_map_at_1000_diff1": 0.7290871254473019, + "nauc_map_at_1000_max": 0.6120752280056199, + "nauc_map_at_1000_std": 0.1339926740717831, + "nauc_map_at_100_diff1": 0.729116189000848, + "nauc_map_at_100_max": 0.6122070042437942, + "nauc_map_at_100_std": 0.13424625550910035, + "nauc_map_at_10_diff1": 0.7275587072696231, + "nauc_map_at_10_max": 0.6131427813295653, + "nauc_map_at_10_std": 0.13247460462832822, + "nauc_map_at_1_diff1": 0.7463322614632609, + "nauc_map_at_1_max": 0.5806226318361674, + "nauc_map_at_1_std": 0.10438451063582097, + "nauc_map_at_20_diff1": 0.7287129282021074, + "nauc_map_at_20_max": 0.6129948863589249, + "nauc_map_at_20_std": 0.13516965016715282, + "nauc_map_at_3_diff1": 0.7276166067316586, + "nauc_map_at_3_max": 0.6078307595298069, + "nauc_map_at_3_std": 0.12362603641270917, + "nauc_map_at_5_diff1": 0.7281564633744259, + "nauc_map_at_5_max": 0.6130927975568831, + "nauc_map_at_5_std": 0.12986554015100024, + "nauc_mrr_at_1000_diff1": 0.7290871254473019, + "nauc_mrr_at_1000_max": 0.6120752280056199, + "nauc_mrr_at_1000_std": 0.1339926740717831, + "nauc_mrr_at_100_diff1": 0.729116189000848, + "nauc_mrr_at_100_max": 0.6122070042437942, + "nauc_mrr_at_100_std": 0.13424625550910035, + "nauc_mrr_at_10_diff1": 0.7275587072696231, + "nauc_mrr_at_10_max": 0.6131427813295653, + "nauc_mrr_at_10_std": 0.13247460462832822, + "nauc_mrr_at_1_diff1": 0.7463322614632609, + "nauc_mrr_at_1_max": 0.5806226318361674, + "nauc_mrr_at_1_std": 0.10438451063582097, + "nauc_mrr_at_20_diff1": 0.7287129282021074, + "nauc_mrr_at_20_max": 0.6129948863589249, + "nauc_mrr_at_20_std": 0.13516965016715282, + "nauc_mrr_at_3_diff1": 0.7276166067316586, + "nauc_mrr_at_3_max": 0.6078307595298069, + "nauc_mrr_at_3_std": 0.12362603641270917, + "nauc_mrr_at_5_diff1": 0.7281564633744259, + "nauc_mrr_at_5_max": 0.6130927975568831, + "nauc_mrr_at_5_std": 0.12986554015100024, + "nauc_ndcg_at_1000_diff1": 0.7267113760326085, + "nauc_ndcg_at_1000_max": 0.6205977818229345, + "nauc_ndcg_at_1000_std": 0.14558789880807216, + "nauc_ndcg_at_100_diff1": 0.727248678404592, + "nauc_ndcg_at_100_max": 0.6240844070904952, + "nauc_ndcg_at_100_std": 0.15255848317430523, + "nauc_ndcg_at_10_diff1": 0.7195813943287325, + "nauc_ndcg_at_10_max": 0.6312960344952803, + "nauc_ndcg_at_10_std": 0.1500285742121337, + "nauc_ndcg_at_1_diff1": 0.7463322614632609, + "nauc_ndcg_at_1_max": 0.5806226318361674, + "nauc_ndcg_at_1_std": 0.10438451063582097, + "nauc_ndcg_at_20_diff1": 0.7237711267067128, + "nauc_ndcg_at_20_max": 0.6309783007730462, + "nauc_ndcg_at_20_std": 0.16121216549752765, + "nauc_ndcg_at_3_diff1": 0.7206010395502354, + "nauc_ndcg_at_3_max": 0.619345909718247, + "nauc_ndcg_at_3_std": 0.13020291795367483, + "nauc_ndcg_at_5_diff1": 0.7210205657545173, + "nauc_ndcg_at_5_max": 0.6303298538413662, + "nauc_ndcg_at_5_std": 0.14244007171848722, + "nauc_precision_at_1000_diff1": 0.8513849984438441, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.7822561123214696, + "nauc_precision_at_100_max": 0.968357713455748, + "nauc_precision_at_100_std": 0.8572120206107152, + "nauc_precision_at_10_diff1": 0.6544594363567019, + "nauc_precision_at_10_max": 0.7857274365145102, + "nauc_precision_at_10_std": 0.30422403703265183, + "nauc_precision_at_1_diff1": 0.7463322614632609, + "nauc_precision_at_1_max": 0.5806226318361674, + "nauc_precision_at_1_std": 0.10438451063582097, + "nauc_precision_at_20_diff1": 0.6694002902365723, + "nauc_precision_at_20_max": 0.8863802549132097, + "nauc_precision_at_20_std": 0.5763220951030971, + "nauc_precision_at_3_diff1": 0.6924928185360182, + "nauc_precision_at_3_max": 0.6660863314034838, + "nauc_precision_at_3_std": 0.15626155616629744, + "nauc_precision_at_5_diff1": 0.6826023673900811, + "nauc_precision_at_5_max": 0.7265547008045097, + "nauc_precision_at_5_std": 0.21227234213903803, + "nauc_recall_at_1000_diff1": 0.8513849984438709, + "nauc_recall_at_1000_max": 1.0, + "nauc_recall_at_1000_std": 1.0, + "nauc_recall_at_100_diff1": 0.7822561123214679, + "nauc_recall_at_100_max": 0.9683577134557472, + "nauc_recall_at_100_std": 0.8572120206107111, + "nauc_recall_at_10_diff1": 0.6544594363567012, + "nauc_recall_at_10_max": 0.7857274365145103, + "nauc_recall_at_10_std": 0.30422403703265355, + "nauc_recall_at_1_diff1": 0.7463322614632609, + "nauc_recall_at_1_max": 0.5806226318361674, + "nauc_recall_at_1_std": 0.10438451063582097, + "nauc_recall_at_20_diff1": 0.6694002902365792, + "nauc_recall_at_20_max": 0.8863802549132098, + "nauc_recall_at_20_std": 0.5763220951031016, + "nauc_recall_at_3_diff1": 0.6924928185360167, + "nauc_recall_at_3_max": 0.6660863314034848, + "nauc_recall_at_3_std": 0.15626155616629717, + "nauc_recall_at_5_diff1": 0.6826023673900833, + "nauc_recall_at_5_max": 0.7265547008045103, + "nauc_recall_at_5_std": 0.21227234213903992, + "ndcg_at_1": 0.6755, + "ndcg_at_10": 0.80482, + "ndcg_at_100": 0.81775, + "ndcg_at_1000": 0.81937, + "ndcg_at_20": 0.81246, + "ndcg_at_3": 0.7689, + "ndcg_at_5": 0.79096, + "precision_at_1": 0.6755, + "precision_at_10": 0.0929, + "precision_at_100": 0.00987, + "precision_at_1000": 0.001, + "precision_at_20": 0.04792, + "precision_at_3": 0.27783, + "precision_at_5": 0.1774, + "recall_at_1": 0.6755, + "recall_at_10": 0.929, + "recall_at_100": 0.9865, + "recall_at_1000": 0.9985, + "recall_at_20": 0.9585, + "recall_at_3": 0.8335, + "recall_at_5": 0.887 + }, + { + "cv_recall_at_1": 0.8125, + "cv_recall_at_10": 0.979, + "cv_recall_at_100": 0.997, + "cv_recall_at_1000": 0.9995, + "cv_recall_at_20": 0.9905, + "cv_recall_at_3": 0.9245, + "cv_recall_at_5": 0.9505, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ], + "main_score": 0.89914, + "map_at_1": 0.8125, + "map_at_10": 0.87317, + "map_at_100": 0.87417, + "map_at_1000": 0.87417, + "map_at_20": 0.87394, + "map_at_3": 0.86325, + "map_at_5": 0.86925, + "mrr_at_1": 0.8125, + "mrr_at_10": 0.873170634920634, + "mrr_at_100": 0.874165591919412, + "mrr_at_1000": 0.8741744359007023, + "mrr_at_20": 0.8739376106628999, + "mrr_at_3": 0.8632499999999987, + "mrr_at_5": 0.8692499999999992, + "nauc_cv_recall_at_1000_diff1": 0.869281045751564, + "nauc_cv_recall_at_1000_max": 1.0, + "nauc_cv_recall_at_1000_std": 1.0, + "nauc_cv_recall_at_100_diff1": 0.8175381263616397, + "nauc_cv_recall_at_100_max": 0.9782135076252607, + "nauc_cv_recall_at_100_std": 0.09025832555244799, + "nauc_cv_recall_at_10_diff1": 0.8532746431906091, + "nauc_cv_recall_at_10_max": 0.8753390244986885, + "nauc_cv_recall_at_10_std": -0.21299630963496155, + "nauc_cv_recall_at_1_diff1": 0.8630334769638712, + "nauc_cv_recall_at_1_max": 0.636707988067617, + "nauc_cv_recall_at_1_std": -0.35566721909181354, + "nauc_cv_recall_at_20_diff1": 0.8380755811096317, + "nauc_cv_recall_at_20_max": 0.9094550100742143, + "nauc_cv_recall_at_20_std": -0.16393925991450467, + "nauc_cv_recall_at_3_diff1": 0.7887318282721475, + "nauc_cv_recall_at_3_max": 0.6857210875520191, + "nauc_cv_recall_at_3_std": -0.40794021802981484, + "nauc_cv_recall_at_5_diff1": 0.8004838298955943, + "nauc_cv_recall_at_5_max": 0.7276311197879799, + "nauc_cv_recall_at_5_std": -0.4019230587858072, + "nauc_map_at_1000_diff1": 0.8475728663389887, + "nauc_map_at_1000_max": 0.6502781025937084, + "nauc_map_at_1000_std": -0.36506027547060665, + "nauc_map_at_100_diff1": 0.8475672508185738, + "nauc_map_at_100_max": 0.6503002284229283, + "nauc_map_at_100_std": -0.36505931191358815, + "nauc_map_at_10_diff1": 0.8476975751166981, + "nauc_map_at_10_max": 0.6519234592343327, + "nauc_map_at_10_std": -0.3643551754692845, + "nauc_map_at_1_diff1": 0.8630334769638712, + "nauc_map_at_1_max": 0.636707988067617, + "nauc_map_at_1_std": -0.35566721909181354, + "nauc_map_at_20_diff1": 0.8475754154618236, + "nauc_map_at_20_max": 0.6507046372651616, + "nauc_map_at_20_std": -0.36490811352349883, + "nauc_map_at_3_diff1": 0.8411035099411375, + "nauc_map_at_3_max": 0.6487267627438167, + "nauc_map_at_3_std": -0.3728350091644064, + "nauc_map_at_5_diff1": 0.8448914569338772, + "nauc_map_at_5_max": 0.6511609163961992, + "nauc_map_at_5_std": -0.36969059260196196, + "nauc_mrr_at_1000_diff1": 0.8475728663389887, + "nauc_mrr_at_1000_max": 0.6502781025937084, + "nauc_mrr_at_1000_std": -0.36506027547060665, + "nauc_mrr_at_100_diff1": 0.8475672508185738, + "nauc_mrr_at_100_max": 0.6503002284229283, + "nauc_mrr_at_100_std": -0.36505931191358815, + "nauc_mrr_at_10_diff1": 0.8476975751166981, + "nauc_mrr_at_10_max": 0.6519234592343327, + "nauc_mrr_at_10_std": -0.3643551754692845, + "nauc_mrr_at_1_diff1": 0.8630334769638712, + "nauc_mrr_at_1_max": 0.636707988067617, + "nauc_mrr_at_1_std": -0.35566721909181354, + "nauc_mrr_at_20_diff1": 0.8475754154618236, + "nauc_mrr_at_20_max": 0.6507046372651616, + "nauc_mrr_at_20_std": -0.36490811352349883, + "nauc_mrr_at_3_diff1": 0.8411035099411375, + "nauc_mrr_at_3_max": 0.6487267627438167, + "nauc_mrr_at_3_std": -0.3728350091644064, + "nauc_mrr_at_5_diff1": 0.8448914569338772, + "nauc_mrr_at_5_max": 0.6511609163961992, + "nauc_mrr_at_5_std": -0.36969059260196196, + "nauc_ndcg_at_1000_diff1": 0.8463031464137469, + "nauc_ndcg_at_1000_max": 0.6541835788530691, + "nauc_ndcg_at_1000_std": -0.3627001519386867, + "nauc_ndcg_at_100_diff1": 0.8461539015610023, + "nauc_ndcg_at_100_max": 0.6552171012623584, + "nauc_ndcg_at_100_std": -0.3619261331869923, + "nauc_ndcg_at_10_diff1": 0.8467822882186511, + "nauc_ndcg_at_10_max": 0.663866200724262, + "nauc_ndcg_at_10_std": -0.35788015610205864, + "nauc_ndcg_at_1_diff1": 0.8630334769638712, + "nauc_ndcg_at_1_max": 0.636707988067617, + "nauc_ndcg_at_1_std": -0.35566721909181354, + "nauc_ndcg_at_20_diff1": 0.8461883899116863, + "nauc_ndcg_at_20_max": 0.6582925692614663, + "nauc_ndcg_at_20_std": -0.36084713873893653, + "nauc_ndcg_at_3_diff1": 0.8322438678494426, + "nauc_ndcg_at_3_max": 0.6548607059305832, + "nauc_ndcg_at_3_std": -0.3788620814328986, + "nauc_ndcg_at_5_diff1": 0.8391437301727366, + "nauc_ndcg_at_5_max": 0.6602246180383025, + "nauc_ndcg_at_5_std": -0.3738041857189095, + "nauc_precision_at_1000_diff1": 0.8692810457516023, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.8175381263616008, + "nauc_precision_at_100_max": 0.9782135076252637, + "nauc_precision_at_100_std": 0.09025832555238163, + "nauc_precision_at_10_diff1": 0.8532746431906064, + "nauc_precision_at_10_max": 0.8753390244986764, + "nauc_precision_at_10_std": -0.21299630963496555, + "nauc_precision_at_1_diff1": 0.8630334769638712, + "nauc_precision_at_1_max": 0.636707988067617, + "nauc_precision_at_1_std": -0.35566721909181354, + "nauc_precision_at_20_diff1": 0.8380755811096442, + "nauc_precision_at_20_max": 0.9094550100741999, + "nauc_precision_at_20_std": -0.16393925991451266, + "nauc_precision_at_3_diff1": 0.7887318282721495, + "nauc_precision_at_3_max": 0.6857210875520209, + "nauc_precision_at_3_std": -0.40794021802981506, + "nauc_precision_at_5_diff1": 0.800483829895592, + "nauc_precision_at_5_max": 0.7276311197879809, + "nauc_precision_at_5_std": -0.40192305878580603, + "nauc_recall_at_1000_diff1": 0.869281045751564, + "nauc_recall_at_1000_max": 1.0, + "nauc_recall_at_1000_std": 1.0, + "nauc_recall_at_100_diff1": 0.8175381263616397, + "nauc_recall_at_100_max": 0.9782135076252607, + "nauc_recall_at_100_std": 0.09025832555244799, + "nauc_recall_at_10_diff1": 0.8532746431906091, + "nauc_recall_at_10_max": 0.8753390244986885, + "nauc_recall_at_10_std": -0.21299630963496155, + "nauc_recall_at_1_diff1": 0.8630334769638712, + "nauc_recall_at_1_max": 0.636707988067617, + "nauc_recall_at_1_std": -0.35566721909181354, + "nauc_recall_at_20_diff1": 0.8380755811096317, + "nauc_recall_at_20_max": 0.9094550100742143, + "nauc_recall_at_20_std": -0.16393925991450467, + "nauc_recall_at_3_diff1": 0.7887318282721475, + "nauc_recall_at_3_max": 0.6857210875520191, + "nauc_recall_at_3_std": -0.40794021802981484, + "nauc_recall_at_5_diff1": 0.8004838298955943, + "nauc_recall_at_5_max": 0.7276311197879799, + "nauc_recall_at_5_std": -0.4019230587858072, + "ndcg_at_1": 0.8125, + "ndcg_at_10": 0.89914, + "ndcg_at_100": 0.90333, + "ndcg_at_1000": 0.90363, + "ndcg_at_20": 0.90201, + "ndcg_at_3": 0.87904, + "ndcg_at_5": 0.8898, + "precision_at_1": 0.8125, + "precision_at_10": 0.0979, + "precision_at_100": 0.00997, + "precision_at_1000": 0.001, + "precision_at_20": 0.04952, + "precision_at_3": 0.30817, + "precision_at_5": 0.1901, + "recall_at_1": 0.8125, + "recall_at_10": 0.979, + "recall_at_100": 0.997, + "recall_at_1000": 0.9995, + "recall_at_20": 0.9905, + "recall_at_3": 0.9245, + "recall_at_5": 0.9505 + }, + { + "cv_recall_at_1": 0.738, + "cv_recall_at_10": 0.9655, + "cv_recall_at_100": 0.999, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.9835, + "cv_recall_at_3": 0.902, + "cv_recall_at_5": 0.936, + "hf_subset": "es", + "languages": [ + "spa-Latn" + ], + "main_score": 0.85832, + "map_at_1": 0.738, + "map_at_10": 0.82313, + "map_at_100": 0.82485, + "map_at_1000": 0.82485, + "map_at_20": 0.82436, + "map_at_3": 0.81117, + "map_at_5": 0.81909, + "mrr_at_1": 0.738, + "mrr_at_10": 0.8231287698412685, + "mrr_at_100": 0.8248457880053297, + "mrr_at_1000": 0.8248512406319713, + "mrr_at_20": 0.8243573382285052, + "mrr_at_3": 0.8111666666666654, + "mrr_at_5": 0.8190916666666657, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.8611111111111184, + "nauc_cv_recall_at_100_max": 1.0, + "nauc_cv_recall_at_100_std": 0.8692810457516946, + "nauc_cv_recall_at_10_diff1": 0.7032774462441969, + "nauc_cv_recall_at_10_max": 0.847054763934563, + "nauc_cv_recall_at_10_std": 0.21517882515325232, + "nauc_cv_recall_at_1_diff1": 0.8228027015155729, + "nauc_cv_recall_at_1_max": 0.657963506973408, + "nauc_cv_recall_at_1_std": 0.07059039237257138, + "nauc_cv_recall_at_20_diff1": 0.6693687576040491, + "nauc_cv_recall_at_20_max": 0.820360467419293, + "nauc_cv_recall_at_20_std": 0.33644568938686364, + "nauc_cv_recall_at_3_diff1": 0.7436331675527346, + "nauc_cv_recall_at_3_max": 0.7652084643381168, + "nauc_cv_recall_at_3_std": 0.13750738390594427, + "nauc_cv_recall_at_5_diff1": 0.7467903828197938, + "nauc_cv_recall_at_5_max": 0.7998694269374397, + "nauc_cv_recall_at_5_std": 0.16819852941176452, + "nauc_map_at_1000_diff1": 0.8055201985370324, + "nauc_map_at_1000_max": 0.6868428387023021, + "nauc_map_at_1000_std": 0.09455811602935846, + "nauc_map_at_100_diff1": 0.8055226699057262, + "nauc_map_at_100_max": 0.686853584905029, + "nauc_map_at_100_std": 0.09458470117793472, + "nauc_map_at_10_diff1": 0.804476460367567, + "nauc_map_at_10_max": 0.6886359840382733, + "nauc_map_at_10_std": 0.09505039704659991, + "nauc_map_at_1_diff1": 0.8228027015155729, + "nauc_map_at_1_max": 0.657963506973408, + "nauc_map_at_1_std": 0.07059039237257138, + "nauc_map_at_20_diff1": 0.8050762144661993, + "nauc_map_at_20_max": 0.687217723852823, + "nauc_map_at_20_std": 0.09495729170023198, + "nauc_map_at_3_diff1": 0.8013479649548041, + "nauc_map_at_3_max": 0.6900708586722066, + "nauc_map_at_3_std": 0.09348015452204224, + "nauc_map_at_5_diff1": 0.8044246497031612, + "nauc_map_at_5_max": 0.689862580590495, + "nauc_map_at_5_std": 0.09487116744284162, + "nauc_mrr_at_1000_diff1": 0.8055201985370324, + "nauc_mrr_at_1000_max": 0.6868428387023021, + "nauc_mrr_at_1000_std": 0.09455811602935846, + "nauc_mrr_at_100_diff1": 0.8055226699057262, + "nauc_mrr_at_100_max": 0.686853584905029, + "nauc_mrr_at_100_std": 0.09458470117793472, + "nauc_mrr_at_10_diff1": 0.804476460367567, + "nauc_mrr_at_10_max": 0.6886359840382733, + "nauc_mrr_at_10_std": 0.09505039704659991, + "nauc_mrr_at_1_diff1": 0.8228027015155729, + "nauc_mrr_at_1_max": 0.657963506973408, + "nauc_mrr_at_1_std": 0.07059039237257138, + "nauc_mrr_at_20_diff1": 0.8050762144661993, + "nauc_mrr_at_20_max": 0.687217723852823, + "nauc_mrr_at_20_std": 0.09495729170023198, + "nauc_mrr_at_3_diff1": 0.8013479649548041, + "nauc_mrr_at_3_max": 0.6900708586722066, + "nauc_mrr_at_3_std": 0.09348015452204224, + "nauc_mrr_at_5_diff1": 0.8044246497031612, + "nauc_mrr_at_5_max": 0.689862580590495, + "nauc_mrr_at_5_std": 0.09487116744284162, + "nauc_ndcg_at_1000_diff1": 0.8029525661469813, + "nauc_ndcg_at_1000_max": 0.6908828889360142, + "nauc_ndcg_at_1000_std": 0.09881132089648269, + "nauc_ndcg_at_100_diff1": 0.8030192391414134, + "nauc_ndcg_at_100_max": 0.6912190957005266, + "nauc_ndcg_at_100_std": 0.09964931117290805, + "nauc_ndcg_at_10_diff1": 0.7971382738540258, + "nauc_ndcg_at_10_max": 0.7003088811485032, + "nauc_ndcg_at_10_std": 0.10401499946098114, + "nauc_ndcg_at_1_diff1": 0.8228027015155729, + "nauc_ndcg_at_1_max": 0.657963506973408, + "nauc_ndcg_at_1_std": 0.07059039237257138, + "nauc_ndcg_at_20_diff1": 0.7995066079500553, + "nauc_ndcg_at_20_max": 0.6940947622326022, + "nauc_ndcg_at_20_std": 0.10398558209170257, + "nauc_ndcg_at_3_diff1": 0.7917995978691086, + "nauc_ndcg_at_3_max": 0.7026512611495505, + "nauc_ndcg_at_3_std": 0.10099772846271363, + "nauc_ndcg_at_5_diff1": 0.7973058952477493, + "nauc_ndcg_at_5_max": 0.7032016467149081, + "nauc_ndcg_at_5_std": 0.10391207396070531, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.8611111111109864, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 0.8692810457515646, + "nauc_precision_at_10_diff1": 0.703277446244195, + "nauc_precision_at_10_max": 0.8470547639345555, + "nauc_precision_at_10_std": 0.21517882515324474, + "nauc_precision_at_1_diff1": 0.8228027015155729, + "nauc_precision_at_1_max": 0.657963506973408, + "nauc_precision_at_1_std": 0.07059039237257138, + "nauc_precision_at_20_diff1": 0.6693687576040411, + "nauc_precision_at_20_max": 0.82036046741928, + "nauc_precision_at_20_std": 0.33644568938685376, + "nauc_precision_at_3_diff1": 0.7436331675527355, + "nauc_precision_at_3_max": 0.7652084643381156, + "nauc_precision_at_3_std": 0.13750738390594283, + "nauc_precision_at_5_diff1": 0.7467903828197912, + "nauc_precision_at_5_max": 0.7998694269374391, + "nauc_precision_at_5_std": 0.1681985294117573, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.8611111111111184, + "nauc_recall_at_100_max": 1.0, + "nauc_recall_at_100_std": 0.8692810457516946, + "nauc_recall_at_10_diff1": 0.7032774462441969, + "nauc_recall_at_10_max": 0.847054763934563, + "nauc_recall_at_10_std": 0.21517882515325232, + "nauc_recall_at_1_diff1": 0.8228027015155729, + "nauc_recall_at_1_max": 0.657963506973408, + "nauc_recall_at_1_std": 0.07059039237257138, + "nauc_recall_at_20_diff1": 0.6693687576040491, + "nauc_recall_at_20_max": 0.820360467419293, + "nauc_recall_at_20_std": 0.33644568938686364, + "nauc_recall_at_3_diff1": 0.7436331675527346, + "nauc_recall_at_3_max": 0.7652084643381168, + "nauc_recall_at_3_std": 0.13750738390594427, + "nauc_recall_at_5_diff1": 0.7467903828197938, + "nauc_recall_at_5_max": 0.7998694269374397, + "nauc_recall_at_5_std": 0.16819852941176452, + "ndcg_at_1": 0.738, + "ndcg_at_10": 0.85832, + "ndcg_at_100": 0.86589, + "ndcg_at_1000": 0.86603, + "ndcg_at_20": 0.86284, + "ndcg_at_3": 0.83453, + "ndcg_at_5": 0.84867, + "precision_at_1": 0.738, + "precision_at_10": 0.09655, + "precision_at_100": 0.00999, + "precision_at_1000": 0.001, + "precision_at_20": 0.04917, + "precision_at_3": 0.30067, + "precision_at_5": 0.1872, + "recall_at_1": 0.738, + "recall_at_10": 0.9655, + "recall_at_100": 0.999, + "recall_at_1000": 1.0, + "recall_at_20": 0.9835, + "recall_at_3": 0.902, + "recall_at_5": 0.936 + }, + { + "cv_recall_at_1": 0.57, + "cv_recall_at_10": 0.89, + "cv_recall_at_100": 0.9885, + "cv_recall_at_1000": 0.998, + "cv_recall_at_20": 0.936, + "cv_recall_at_3": 0.756, + "cv_recall_at_5": 0.8245, + "hf_subset": "id", + "languages": [ + "ind-Latn" + ], + "main_score": 0.72935, + "map_at_1": 0.57, + "map_at_10": 0.67785, + "map_at_100": 0.68246, + "map_at_1000": 0.68253, + "map_at_20": 0.68098, + "map_at_3": 0.65292, + "map_at_5": 0.66864, + "mrr_at_1": 0.57, + "mrr_at_10": 0.6778513888888882, + "mrr_at_100": 0.682463976758287, + "mrr_at_1000": 0.6825288641722557, + "mrr_at_20": 0.6809768672568393, + "mrr_at_3": 0.6529166666666663, + "mrr_at_5": 0.6686416666666668, + "nauc_cv_recall_at_1000_diff1": 0.057773109243720255, + "nauc_cv_recall_at_1000_max": 0.07119514472454563, + "nauc_cv_recall_at_1000_std": 0.3162931839401999, + "nauc_cv_recall_at_100_diff1": 0.5459343157552812, + "nauc_cv_recall_at_100_max": 0.6831283237932791, + "nauc_cv_recall_at_100_std": 0.4349449924897459, + "nauc_cv_recall_at_10_diff1": 0.506069066231423, + "nauc_cv_recall_at_10_max": 0.6634932565930765, + "nauc_cv_recall_at_10_std": 0.18059230306674634, + "nauc_cv_recall_at_1_diff1": 0.7044065096654757, + "nauc_cv_recall_at_1_max": 0.5750721143496796, + "nauc_cv_recall_at_1_std": 0.046992006429299085, + "nauc_cv_recall_at_20_diff1": 0.5504201680672272, + "nauc_cv_recall_at_20_max": 0.7070786647992531, + "nauc_cv_recall_at_20_std": 0.2757972980858999, + "nauc_cv_recall_at_3_diff1": 0.5681348565963951, + "nauc_cv_recall_at_3_max": 0.6170784343861266, + "nauc_cv_recall_at_3_std": 0.09851473313011817, + "nauc_cv_recall_at_5_diff1": 0.5156420439651506, + "nauc_cv_recall_at_5_max": 0.6434716064276265, + "nauc_cv_recall_at_5_std": 0.146063346120565, + "nauc_map_at_1000_diff1": 0.6561740158631391, + "nauc_map_at_1000_max": 0.5903652066345317, + "nauc_map_at_1000_std": 0.06752440959391125, + "nauc_map_at_100_diff1": 0.6561738489049953, + "nauc_map_at_100_max": 0.5904283966716123, + "nauc_map_at_100_std": 0.0676291858616124, + "nauc_map_at_10_diff1": 0.65333134434392, + "nauc_map_at_10_max": 0.5913969782544157, + "nauc_map_at_10_std": 0.06836479826993805, + "nauc_map_at_1_diff1": 0.7044065096654757, + "nauc_map_at_1_max": 0.5750721143496796, + "nauc_map_at_1_std": 0.046992006429299085, + "nauc_map_at_20_diff1": 0.6556880678927408, + "nauc_map_at_20_max": 0.5909962160263641, + "nauc_map_at_20_std": 0.06838331967377713, + "nauc_map_at_3_diff1": 0.653394169001542, + "nauc_map_at_3_max": 0.5888483426159291, + "nauc_map_at_3_std": 0.06312086141745396, + "nauc_map_at_5_diff1": 0.6494305220897459, + "nauc_map_at_5_max": 0.5912390189784108, + "nauc_map_at_5_std": 0.06860456750834616, + "nauc_mrr_at_1000_diff1": 0.6561740158631391, + "nauc_mrr_at_1000_max": 0.5903652066345317, + "nauc_mrr_at_1000_std": 0.06752440959391125, + "nauc_mrr_at_100_diff1": 0.6561738489049953, + "nauc_mrr_at_100_max": 0.5904283966716123, + "nauc_mrr_at_100_std": 0.0676291858616124, + "nauc_mrr_at_10_diff1": 0.65333134434392, + "nauc_mrr_at_10_max": 0.5913969782544157, + "nauc_mrr_at_10_std": 0.06836479826993805, + "nauc_mrr_at_1_diff1": 0.7044065096654757, + "nauc_mrr_at_1_max": 0.5750721143496796, + "nauc_mrr_at_1_std": 0.046992006429299085, + "nauc_mrr_at_20_diff1": 0.6556880678927408, + "nauc_mrr_at_20_max": 0.5909962160263641, + "nauc_mrr_at_20_std": 0.06838331967377713, + "nauc_mrr_at_3_diff1": 0.653394169001542, + "nauc_mrr_at_3_max": 0.5888483426159291, + "nauc_mrr_at_3_std": 0.06312086141745396, + "nauc_mrr_at_5_diff1": 0.6494305220897459, + "nauc_mrr_at_5_max": 0.5912390189784108, + "nauc_mrr_at_5_std": 0.06860456750834616, + "nauc_ndcg_at_1000_diff1": 0.6483006530425621, + "nauc_ndcg_at_1000_max": 0.5933424088703956, + "nauc_ndcg_at_1000_std": 0.07405067389762883, + "nauc_ndcg_at_100_diff1": 0.6483197398489603, + "nauc_ndcg_at_100_max": 0.5947897242846063, + "nauc_ndcg_at_100_std": 0.07655442331368115, + "nauc_ndcg_at_10_diff1": 0.6340715926512107, + "nauc_ndcg_at_10_max": 0.600450440109288, + "nauc_ndcg_at_10_std": 0.08208340679131955, + "nauc_ndcg_at_1_diff1": 0.7044065096654757, + "nauc_ndcg_at_1_max": 0.5750721143496796, + "nauc_ndcg_at_1_std": 0.046992006429299085, + "nauc_ndcg_at_20_diff1": 0.6440067539607353, + "nauc_ndcg_at_20_max": 0.5999888493818042, + "nauc_ndcg_at_20_std": 0.083672066995679, + "nauc_ndcg_at_3_diff1": 0.6350601872400334, + "nauc_ndcg_at_3_max": 0.5948277431038169, + "nauc_ndcg_at_3_std": 0.07058416162338209, + "nauc_ndcg_at_5_diff1": 0.6259232001622383, + "nauc_ndcg_at_5_max": 0.6001942413051717, + "nauc_ndcg_at_5_std": 0.08181067673621384, + "nauc_precision_at_1000_diff1": 0.05777310924364636, + "nauc_precision_at_1000_max": 0.07119514472449866, + "nauc_precision_at_1000_std": 0.31629318394019806, + "nauc_precision_at_100_diff1": 0.5459343157552841, + "nauc_precision_at_100_max": 0.6831283237932635, + "nauc_precision_at_100_std": 0.4349449924897548, + "nauc_precision_at_10_diff1": 0.5060690662314247, + "nauc_precision_at_10_max": 0.6634932565930743, + "nauc_precision_at_10_std": 0.18059230306674617, + "nauc_precision_at_1_diff1": 0.7044065096654757, + "nauc_precision_at_1_max": 0.5750721143496796, + "nauc_precision_at_1_std": 0.046992006429299085, + "nauc_precision_at_20_diff1": 0.5504201680672255, + "nauc_precision_at_20_max": 0.7070786647992503, + "nauc_precision_at_20_std": 0.2757972980858991, + "nauc_precision_at_3_diff1": 0.5681348565963955, + "nauc_precision_at_3_max": 0.617078434386128, + "nauc_precision_at_3_std": 0.09851473313011903, + "nauc_precision_at_5_diff1": 0.5156420439651481, + "nauc_precision_at_5_max": 0.6434716064276249, + "nauc_precision_at_5_std": 0.14606334612056285, + "nauc_recall_at_1000_diff1": 0.057773109243720255, + "nauc_recall_at_1000_max": 0.07119514472454563, + "nauc_recall_at_1000_std": 0.3162931839401999, + "nauc_recall_at_100_diff1": 0.5459343157552812, + "nauc_recall_at_100_max": 0.6831283237932791, + "nauc_recall_at_100_std": 0.4349449924897459, + "nauc_recall_at_10_diff1": 0.506069066231423, + "nauc_recall_at_10_max": 0.6634932565930765, + "nauc_recall_at_10_std": 0.18059230306674634, + "nauc_recall_at_1_diff1": 0.7044065096654757, + "nauc_recall_at_1_max": 0.5750721143496796, + "nauc_recall_at_1_std": 0.046992006429299085, + "nauc_recall_at_20_diff1": 0.5504201680672272, + "nauc_recall_at_20_max": 0.7070786647992531, + "nauc_recall_at_20_std": 0.2757972980858999, + "nauc_recall_at_3_diff1": 0.5681348565963951, + "nauc_recall_at_3_max": 0.6170784343861266, + "nauc_recall_at_3_std": 0.09851473313011817, + "nauc_recall_at_5_diff1": 0.5156420439651506, + "nauc_recall_at_5_max": 0.6434716064276265, + "nauc_recall_at_5_std": 0.146063346120565, + "ndcg_at_1": 0.57, + "ndcg_at_10": 0.72935, + "ndcg_at_100": 0.75091, + "ndcg_at_1000": 0.75221, + "ndcg_at_20": 0.7409, + "ndcg_at_3": 0.67943, + "ndcg_at_5": 0.70771, + "precision_at_1": 0.57, + "precision_at_10": 0.089, + "precision_at_100": 0.00989, + "precision_at_1000": 0.001, + "precision_at_20": 0.0468, + "precision_at_3": 0.252, + "precision_at_5": 0.1649, + "recall_at_1": 0.57, + "recall_at_10": 0.89, + "recall_at_100": 0.9885, + "recall_at_1000": 0.998, + "recall_at_20": 0.936, + "recall_at_3": 0.756, + "recall_at_5": 0.8245 + }, + { + "cv_recall_at_1": 0.6515, + "cv_recall_at_10": 0.9325, + "cv_recall_at_100": 0.9915, + "cv_recall_at_1000": 0.999, + "cv_recall_at_20": 0.962, + "cv_recall_at_3": 0.822, + "cv_recall_at_5": 0.879, + "hf_subset": "ja", + "languages": [ + "jpn-Jpan" + ], + "main_score": 0.79245, + "map_at_1": 0.6515, + "map_at_10": 0.74747, + "map_at_100": 0.75038, + "map_at_1000": 0.75042, + "map_at_20": 0.74956, + "map_at_3": 0.72733, + "map_at_5": 0.74018, + "mrr_at_1": 0.6515, + "mrr_at_10": 0.7474712301587296, + "mrr_at_100": 0.7503837624364388, + "mrr_at_1000": 0.7504248142935365, + "mrr_at_20": 0.7495552861139542, + "mrr_at_3": 0.7273333333333322, + "mrr_at_5": 0.740183333333333, + "nauc_cv_recall_at_1000_diff1": 0.7770774976656952, + "nauc_cv_recall_at_1000_max": 1.0, + "nauc_cv_recall_at_1000_std": 1.0, + "nauc_cv_recall_at_100_diff1": 0.8009007524578522, + "nauc_cv_recall_at_100_max": 0.9682814302191535, + "nauc_cv_recall_at_100_std": 0.729856648541773, + "nauc_cv_recall_at_10_diff1": 0.5883217484524672, + "nauc_cv_recall_at_10_max": 0.7557526714389453, + "nauc_cv_recall_at_10_std": 0.3104955562471906, + "nauc_cv_recall_at_1_diff1": 0.7329375789067353, + "nauc_cv_recall_at_1_max": 0.5361300015750745, + "nauc_cv_recall_at_1_std": 0.06148747407860173, + "nauc_cv_recall_at_20_diff1": 0.623163300407883, + "nauc_cv_recall_at_20_max": 0.8448633839500713, + "nauc_cv_recall_at_20_std": 0.42984298982750796, + "nauc_cv_recall_at_3_diff1": 0.6402739291556773, + "nauc_cv_recall_at_3_max": 0.6452442159383034, + "nauc_cv_recall_at_3_std": 0.17984982027398486, + "nauc_cv_recall_at_5_diff1": 0.6157551065526523, + "nauc_cv_recall_at_5_max": 0.6875468807984149, + "nauc_cv_recall_at_5_std": 0.28306329993446605, + "nauc_map_at_1000_diff1": 0.7039322369949266, + "nauc_map_at_1000_max": 0.5719568116443199, + "nauc_map_at_1000_std": 0.10175653818610049, + "nauc_map_at_100_diff1": 0.7039442745246894, + "nauc_map_at_100_max": 0.5720300068067742, + "nauc_map_at_100_std": 0.10188202291296097, + "nauc_map_at_10_diff1": 0.7021694520509648, + "nauc_map_at_10_max": 0.5735604207840197, + "nauc_map_at_10_std": 0.10327802325986768, + "nauc_map_at_1_diff1": 0.7329375789067353, + "nauc_map_at_1_max": 0.5361300015750745, + "nauc_map_at_1_std": 0.06148747407860173, + "nauc_map_at_20_diff1": 0.7035120770713532, + "nauc_map_at_20_max": 0.5729577824792697, + "nauc_map_at_20_std": 0.10260589720880942, + "nauc_map_at_3_diff1": 0.7005784793460021, + "nauc_map_at_3_max": 0.5730788546799835, + "nauc_map_at_3_std": 0.10061225940427539, + "nauc_map_at_5_diff1": 0.7005878046876627, + "nauc_map_at_5_max": 0.5743062534656194, + "nauc_map_at_5_std": 0.10807883793605706, + "nauc_mrr_at_1000_diff1": 0.7039322369949266, + "nauc_mrr_at_1000_max": 0.5719568116443199, + "nauc_mrr_at_1000_std": 0.10175653818610049, + "nauc_mrr_at_100_diff1": 0.7039442745246894, + "nauc_mrr_at_100_max": 0.5720300068067742, + "nauc_mrr_at_100_std": 0.10188202291296097, + "nauc_mrr_at_10_diff1": 0.7021694520509648, + "nauc_mrr_at_10_max": 0.5735604207840197, + "nauc_mrr_at_10_std": 0.10327802325986768, + "nauc_mrr_at_1_diff1": 0.7329375789067353, + "nauc_mrr_at_1_max": 0.5361300015750745, + "nauc_mrr_at_1_std": 0.06148747407860173, + "nauc_mrr_at_20_diff1": 0.7035120770713532, + "nauc_mrr_at_20_max": 0.5729577824792697, + "nauc_mrr_at_20_std": 0.10260589720880942, + "nauc_mrr_at_3_diff1": 0.7005784793460021, + "nauc_mrr_at_3_max": 0.5730788546799835, + "nauc_mrr_at_3_std": 0.10061225940427539, + "nauc_mrr_at_5_diff1": 0.7005878046876627, + "nauc_mrr_at_5_max": 0.5743062534656194, + "nauc_mrr_at_5_std": 0.10807883793605706, + "nauc_ndcg_at_1000_diff1": 0.700087104488326, + "nauc_ndcg_at_1000_max": 0.5795021097985256, + "nauc_ndcg_at_1000_std": 0.11154540403613405, + "nauc_ndcg_at_100_diff1": 0.7006550596100719, + "nauc_ndcg_at_100_max": 0.5817496273763957, + "nauc_ndcg_at_100_std": 0.11502416071470394, + "nauc_ndcg_at_10_diff1": 0.6905913110087234, + "nauc_ndcg_at_10_max": 0.5912371201845855, + "nauc_ndcg_at_10_std": 0.1235839893326974, + "nauc_ndcg_at_1_diff1": 0.7329375789067353, + "nauc_ndcg_at_1_max": 0.5361300015750745, + "nauc_ndcg_at_1_std": 0.06148747407860173, + "nauc_ndcg_at_20_diff1": 0.6966587387995632, + "nauc_ndcg_at_20_max": 0.5891614061644422, + "nauc_ndcg_at_20_std": 0.121914444846387, + "nauc_ndcg_at_3_diff1": 0.6884729340233343, + "nauc_ndcg_at_3_max": 0.587509909021513, + "nauc_ndcg_at_3_std": 0.11641242005242053, + "nauc_ndcg_at_5_diff1": 0.6873311264384971, + "nauc_ndcg_at_5_max": 0.5917469354526042, + "nauc_ndcg_at_5_std": 0.13434677089258235, + "nauc_precision_at_1000_diff1": 0.7770774976657188, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.8009007524578535, + "nauc_precision_at_100_max": 0.9682814302191545, + "nauc_precision_at_100_std": 0.7298566485417957, + "nauc_precision_at_10_diff1": 0.5883217484524637, + "nauc_precision_at_10_max": 0.7557526714389439, + "nauc_precision_at_10_std": 0.31049555624718805, + "nauc_precision_at_1_diff1": 0.7329375789067353, + "nauc_precision_at_1_max": 0.5361300015750745, + "nauc_precision_at_1_std": 0.06148747407860173, + "nauc_precision_at_20_diff1": 0.6231633004078801, + "nauc_precision_at_20_max": 0.8448633839500712, + "nauc_precision_at_20_std": 0.4298429898275066, + "nauc_precision_at_3_diff1": 0.6402739291556786, + "nauc_precision_at_3_max": 0.6452442159383038, + "nauc_precision_at_3_std": 0.17984982027398597, + "nauc_precision_at_5_diff1": 0.6157551065526519, + "nauc_precision_at_5_max": 0.6875468807984131, + "nauc_precision_at_5_std": 0.2830632999344636, + "nauc_recall_at_1000_diff1": 0.7770774976656952, + "nauc_recall_at_1000_max": 1.0, + "nauc_recall_at_1000_std": 1.0, + "nauc_recall_at_100_diff1": 0.8009007524578522, + "nauc_recall_at_100_max": 0.9682814302191535, + "nauc_recall_at_100_std": 0.729856648541773, + "nauc_recall_at_10_diff1": 0.5883217484524672, + "nauc_recall_at_10_max": 0.7557526714389453, + "nauc_recall_at_10_std": 0.3104955562471906, + "nauc_recall_at_1_diff1": 0.7329375789067353, + "nauc_recall_at_1_max": 0.5361300015750745, + "nauc_recall_at_1_std": 0.06148747407860173, + "nauc_recall_at_20_diff1": 0.623163300407883, + "nauc_recall_at_20_max": 0.8448633839500713, + "nauc_recall_at_20_std": 0.42984298982750796, + "nauc_recall_at_3_diff1": 0.6402739291556773, + "nauc_recall_at_3_max": 0.6452442159383034, + "nauc_recall_at_3_std": 0.17984982027398486, + "nauc_recall_at_5_diff1": 0.6157551065526523, + "nauc_recall_at_5_max": 0.6875468807984149, + "nauc_recall_at_5_std": 0.28306329993446605, + "ndcg_at_1": 0.6515, + "ndcg_at_10": 0.79245, + "ndcg_at_100": 0.80558, + "ndcg_at_1000": 0.80656, + "ndcg_at_20": 0.79995, + "ndcg_at_3": 0.75168, + "ndcg_at_5": 0.775, + "precision_at_1": 0.6515, + "precision_at_10": 0.09325, + "precision_at_100": 0.00992, + "precision_at_1000": 0.001, + "precision_at_20": 0.0481, + "precision_at_3": 0.274, + "precision_at_5": 0.1758, + "recall_at_1": 0.6515, + "recall_at_10": 0.9325, + "recall_at_100": 0.9915, + "recall_at_1000": 0.999, + "recall_at_20": 0.962, + "recall_at_3": 0.822, + "recall_at_5": 0.879 + }, + { + "cv_recall_at_1": 0.733, + "cv_recall_at_10": 0.963, + "cv_recall_at_100": 0.9985, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.984, + "cv_recall_at_3": 0.88, + "cv_recall_at_5": 0.9275, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.85232, + "map_at_1": 0.733, + "map_at_10": 0.81632, + "map_at_100": 0.81832, + "map_at_1000": 0.81834, + "map_at_20": 0.81786, + "map_at_3": 0.8005, + "map_at_5": 0.81127, + "mrr_at_1": 0.733, + "mrr_at_10": 0.8163236111111105, + "mrr_at_100": 0.8183248956975758, + "mrr_at_1000": 0.8183350849805338, + "mrr_at_20": 0.8178556814509012, + "mrr_at_3": 0.8004999999999989, + "mrr_at_5": 0.8112749999999994, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.8513849984438709, + "nauc_cv_recall_at_100_max": 0.8638344226580088, + "nauc_cv_recall_at_100_std": -0.10768751945235333, + "nauc_cv_recall_at_10_diff1": 0.7755759961642286, + "nauc_cv_recall_at_10_max": 0.7924899689605557, + "nauc_cv_recall_at_10_std": 0.03805486158427406, + "nauc_cv_recall_at_1_diff1": 0.8133792547946724, + "nauc_cv_recall_at_1_max": 0.6252031052536563, + "nauc_cv_recall_at_1_std": 0.09226384736404669, + "nauc_cv_recall_at_20_diff1": 0.7742471988795507, + "nauc_cv_recall_at_20_max": 0.8285189075630218, + "nauc_cv_recall_at_20_std": 0.1130222922502361, + "nauc_cv_recall_at_3_diff1": 0.7380508272351263, + "nauc_cv_recall_at_3_max": 0.7140669742284433, + "nauc_cv_recall_at_3_std": 0.09508232580337304, + "nauc_cv_recall_at_5_diff1": 0.7234907756205932, + "nauc_cv_recall_at_5_max": 0.753079622653659, + "nauc_cv_recall_at_5_std": 0.051019028300975704, + "nauc_map_at_1000_diff1": 0.7952082628577973, + "nauc_map_at_1000_max": 0.6478391020353417, + "nauc_map_at_1000_std": 0.09081338024626912, + "nauc_map_at_100_diff1": 0.7952086285167334, + "nauc_map_at_100_max": 0.6478494920991844, + "nauc_map_at_100_std": 0.09077674916537626, + "nauc_map_at_10_diff1": 0.7951467350309017, + "nauc_map_at_10_max": 0.6496051997722657, + "nauc_map_at_10_std": 0.09048990915348447, + "nauc_map_at_1_diff1": 0.8133792547946724, + "nauc_map_at_1_max": 0.6252031052536563, + "nauc_map_at_1_std": 0.09226384736404669, + "nauc_map_at_20_diff1": 0.7952202923863493, + "nauc_map_at_20_max": 0.6483931037134688, + "nauc_map_at_20_std": 0.09102570610885713, + "nauc_map_at_3_diff1": 0.790142614160109, + "nauc_map_at_3_max": 0.6517459403239515, + "nauc_map_at_3_std": 0.09487554955026746, + "nauc_map_at_5_diff1": 0.7914810063107847, + "nauc_map_at_5_max": 0.651685378725342, + "nauc_map_at_5_std": 0.09063688167845714, + "nauc_mrr_at_1000_diff1": 0.7952082628577973, + "nauc_mrr_at_1000_max": 0.6478391020353417, + "nauc_mrr_at_1000_std": 0.09081338024626912, + "nauc_mrr_at_100_diff1": 0.7952086285167334, + "nauc_mrr_at_100_max": 0.6478494920991844, + "nauc_mrr_at_100_std": 0.09077674916537626, + "nauc_mrr_at_10_diff1": 0.7951467350309017, + "nauc_mrr_at_10_max": 0.6496051997722657, + "nauc_mrr_at_10_std": 0.09048990915348447, + "nauc_mrr_at_1_diff1": 0.8133792547946724, + "nauc_mrr_at_1_max": 0.6252031052536563, + "nauc_mrr_at_1_std": 0.09226384736404669, + "nauc_mrr_at_20_diff1": 0.7952202923863493, + "nauc_mrr_at_20_max": 0.6483931037134688, + "nauc_mrr_at_20_std": 0.09102570610885713, + "nauc_mrr_at_3_diff1": 0.790142614160109, + "nauc_mrr_at_3_max": 0.6517459403239515, + "nauc_mrr_at_3_std": 0.09487554955026746, + "nauc_mrr_at_5_diff1": 0.7914810063107847, + "nauc_mrr_at_5_max": 0.651685378725342, + "nauc_mrr_at_5_std": 0.09063688167845714, + "nauc_ndcg_at_1000_diff1": 0.793184615751275, + "nauc_ndcg_at_1000_max": 0.6512997647156443, + "nauc_ndcg_at_1000_std": 0.0896376656750505, + "nauc_ndcg_at_100_diff1": 0.7932576505906453, + "nauc_ndcg_at_100_max": 0.6516218237126856, + "nauc_ndcg_at_100_std": 0.08916329598012869, + "nauc_ndcg_at_10_diff1": 0.7923981235421121, + "nauc_ndcg_at_10_max": 0.6601701521909391, + "nauc_ndcg_at_10_std": 0.08679745362754197, + "nauc_ndcg_at_1_diff1": 0.8133792547946724, + "nauc_ndcg_at_1_max": 0.6252031052536563, + "nauc_ndcg_at_1_std": 0.09226384736404669, + "nauc_ndcg_at_20_diff1": 0.7928995512374133, + "nauc_ndcg_at_20_max": 0.6555697297862627, + "nauc_ndcg_at_20_std": 0.09046056892469329, + "nauc_ndcg_at_3_diff1": 0.7805316637468971, + "nauc_ndcg_at_3_max": 0.6632016786911084, + "nauc_ndcg_at_3_std": 0.0949989221869438, + "nauc_ndcg_at_5_diff1": 0.7825497211406456, + "nauc_ndcg_at_5_max": 0.664619339432301, + "nauc_ndcg_at_5_std": 0.08604866742140158, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.851384998443846, + "nauc_precision_at_100_max": 0.8638344226578644, + "nauc_precision_at_100_std": -0.10768751945219786, + "nauc_precision_at_10_diff1": 0.7755759961642249, + "nauc_precision_at_10_max": 0.7924899689605511, + "nauc_precision_at_10_std": 0.0380548615842704, + "nauc_precision_at_1_diff1": 0.8133792547946724, + "nauc_precision_at_1_max": 0.6252031052536563, + "nauc_precision_at_1_std": 0.09226384736404669, + "nauc_precision_at_20_diff1": 0.7742471988795416, + "nauc_precision_at_20_max": 0.8285189075630202, + "nauc_precision_at_20_std": 0.11302229225024674, + "nauc_precision_at_3_diff1": 0.7380508272351264, + "nauc_precision_at_3_max": 0.7140669742284463, + "nauc_precision_at_3_std": 0.0950823258033741, + "nauc_precision_at_5_diff1": 0.7234907756205926, + "nauc_precision_at_5_max": 0.7530796226536576, + "nauc_precision_at_5_std": 0.05101902830097392, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.8513849984438709, + "nauc_recall_at_100_max": 0.8638344226580088, + "nauc_recall_at_100_std": -0.10768751945235333, + "nauc_recall_at_10_diff1": 0.7755759961642286, + "nauc_recall_at_10_max": 0.7924899689605557, + "nauc_recall_at_10_std": 0.03805486158427406, + "nauc_recall_at_1_diff1": 0.8133792547946724, + "nauc_recall_at_1_max": 0.6252031052536563, + "nauc_recall_at_1_std": 0.09226384736404669, + "nauc_recall_at_20_diff1": 0.7742471988795507, + "nauc_recall_at_20_max": 0.8285189075630218, + "nauc_recall_at_20_std": 0.1130222922502361, + "nauc_recall_at_3_diff1": 0.7380508272351263, + "nauc_recall_at_3_max": 0.7140669742284433, + "nauc_recall_at_3_std": 0.09508232580337304, + "nauc_recall_at_5_diff1": 0.7234907756205932, + "nauc_recall_at_5_max": 0.753079622653659, + "nauc_recall_at_5_std": 0.051019028300975704, + "ndcg_at_1": 0.733, + "ndcg_at_10": 0.85232, + "ndcg_at_100": 0.86059, + "ndcg_at_1000": 0.8608, + "ndcg_at_20": 0.85772, + "ndcg_at_3": 0.82103, + "ndcg_at_5": 0.84053, + "precision_at_1": 0.733, + "precision_at_10": 0.0963, + "precision_at_100": 0.00999, + "precision_at_1000": 0.001, + "precision_at_20": 0.0492, + "precision_at_3": 0.29333, + "precision_at_5": 0.1855, + "recall_at_1": 0.733, + "recall_at_10": 0.963, + "recall_at_100": 0.9985, + "recall_at_1000": 1.0, + "recall_at_20": 0.984, + "recall_at_3": 0.88, + "recall_at_5": 0.9275 + }, + { + "cv_recall_at_1": 0.329, + "cv_recall_at_10": 0.6625, + "cv_recall_at_100": 0.898, + "cv_recall_at_1000": 0.9935, + "cv_recall_at_20": 0.753, + "cv_recall_at_3": 0.488, + "cv_recall_at_5": 0.566, + "hf_subset": "tr", + "languages": [ + "tur-Latn" + ], + "main_score": 0.48558, + "map_at_1": 0.329, + "map_at_10": 0.43015, + "map_at_100": 0.4402, + "map_at_1000": 0.44068, + "map_at_20": 0.43662, + "map_at_3": 0.39908, + "map_at_5": 0.41716, + "mrr_at_1": 0.329, + "mrr_at_10": 0.4301498015873023, + "mrr_at_100": 0.44020367057678783, + "mrr_at_1000": 0.44068107415161467, + "mrr_at_20": 0.43662267512329517, + "mrr_at_3": 0.3990833333333339, + "mrr_at_5": 0.4171583333333347, + "nauc_cv_recall_at_1000_diff1": 0.570746247216831, + "nauc_cv_recall_at_1000_max": 0.9355383178912546, + "nauc_cv_recall_at_1000_std": 0.8143719026071998, + "nauc_cv_recall_at_100_diff1": 0.28561103871074306, + "nauc_cv_recall_at_100_max": 0.6815610029184481, + "nauc_cv_recall_at_100_std": 0.5854242763532237, + "nauc_cv_recall_at_10_diff1": 0.34808730638537383, + "nauc_cv_recall_at_10_max": 0.6230498863849571, + "nauc_cv_recall_at_10_std": 0.38972169526152306, + "nauc_cv_recall_at_1_diff1": 0.4897268439399369, + "nauc_cv_recall_at_1_max": 0.48143439232866647, + "nauc_cv_recall_at_1_std": 0.22002136550057733, + "nauc_cv_recall_at_20_diff1": 0.3732459116186757, + "nauc_cv_recall_at_20_max": 0.6731810217095842, + "nauc_cv_recall_at_20_std": 0.46552168739127814, + "nauc_cv_recall_at_3_diff1": 0.39113799929941523, + "nauc_cv_recall_at_3_max": 0.5270269194433536, + "nauc_cv_recall_at_3_std": 0.28440694428227187, + "nauc_cv_recall_at_5_diff1": 0.3756919063036706, + "nauc_cv_recall_at_5_max": 0.5573107027224745, + "nauc_cv_recall_at_5_std": 0.3082772997579581, + "nauc_map_at_1000_diff1": 0.4414051282408367, + "nauc_map_at_1000_max": 0.514311675803377, + "nauc_map_at_1000_std": 0.2621652878648845, + "nauc_map_at_100_diff1": 0.4411597090934524, + "nauc_map_at_100_max": 0.5145432570532894, + "nauc_map_at_100_std": 0.2625758722812923, + "nauc_map_at_10_diff1": 0.43856234721121634, + "nauc_map_at_10_max": 0.5144546481309277, + "nauc_map_at_10_std": 0.26100856265121153, + "nauc_map_at_1_diff1": 0.4897268439399369, + "nauc_map_at_1_max": 0.48143439232866647, + "nauc_map_at_1_std": 0.22002136550057733, + "nauc_map_at_20_diff1": 0.44113261525244246, + "nauc_map_at_20_max": 0.5157374897438756, + "nauc_map_at_20_std": 0.2631221937442976, + "nauc_map_at_3_diff1": 0.4430321264763373, + "nauc_map_at_3_max": 0.5016682101912822, + "nauc_map_at_3_std": 0.24791585529023333, + "nauc_map_at_5_diff1": 0.44076808631336867, + "nauc_map_at_5_max": 0.5079389409486393, + "nauc_map_at_5_std": 0.25294942024853534, + "nauc_mrr_at_1000_diff1": 0.4414051282408367, + "nauc_mrr_at_1000_max": 0.514311675803377, + "nauc_mrr_at_1000_std": 0.2621652878648845, + "nauc_mrr_at_100_diff1": 0.4411597090934524, + "nauc_mrr_at_100_max": 0.5145432570532894, + "nauc_mrr_at_100_std": 0.2625758722812923, + "nauc_mrr_at_10_diff1": 0.43856234721121634, + "nauc_mrr_at_10_max": 0.5144546481309277, + "nauc_mrr_at_10_std": 0.26100856265121153, + "nauc_mrr_at_1_diff1": 0.4897268439399369, + "nauc_mrr_at_1_max": 0.48143439232866647, + "nauc_mrr_at_1_std": 0.22002136550057733, + "nauc_mrr_at_20_diff1": 0.44113261525244246, + "nauc_mrr_at_20_max": 0.5157374897438756, + "nauc_mrr_at_20_std": 0.2631221937442976, + "nauc_mrr_at_3_diff1": 0.4430321264763373, + "nauc_mrr_at_3_max": 0.5016682101912822, + "nauc_mrr_at_3_std": 0.24791585529023333, + "nauc_mrr_at_5_diff1": 0.44076808631336867, + "nauc_mrr_at_5_max": 0.5079389409486393, + "nauc_mrr_at_5_std": 0.25294942024853534, + "nauc_ndcg_at_1000_diff1": 0.4330018707890607, + "nauc_ndcg_at_1000_max": 0.5266139850395504, + "nauc_ndcg_at_1000_std": 0.2796469830342363, + "nauc_ndcg_at_100_diff1": 0.4260189469019916, + "nauc_ndcg_at_100_max": 0.5325043437016203, + "nauc_ndcg_at_100_std": 0.2916521593222054, + "nauc_ndcg_at_10_diff1": 0.4190897586950866, + "nauc_ndcg_at_10_max": 0.5363395029642329, + "nauc_ndcg_at_10_std": 0.2870156955251191, + "nauc_ndcg_at_1_diff1": 0.4897268439399369, + "nauc_ndcg_at_1_max": 0.48143439232866647, + "nauc_ndcg_at_1_std": 0.22002136550057733, + "nauc_ndcg_at_20_diff1": 0.4274167281601817, + "nauc_ndcg_at_20_max": 0.5416035681782816, + "nauc_ndcg_at_20_std": 0.2959927566759077, + "nauc_ndcg_at_3_diff1": 0.42958629116833824, + "nauc_ndcg_at_3_max": 0.508177698679707, + "nauc_ndcg_at_3_std": 0.25725414826981835, + "nauc_ndcg_at_5_diff1": 0.42485040568184196, + "nauc_ndcg_at_5_max": 0.5196752086294824, + "nauc_ndcg_at_5_std": 0.26626862328931283, + "nauc_precision_at_1000_diff1": 0.5707462472168311, + "nauc_precision_at_1000_max": 0.9355383178912654, + "nauc_precision_at_1000_std": 0.8143719026071755, + "nauc_precision_at_100_diff1": 0.28561103871074167, + "nauc_precision_at_100_max": 0.6815610029184485, + "nauc_precision_at_100_std": 0.5854242763532232, + "nauc_precision_at_10_diff1": 0.3480873063853732, + "nauc_precision_at_10_max": 0.6230498863849561, + "nauc_precision_at_10_std": 0.3897216952615221, + "nauc_precision_at_1_diff1": 0.4897268439399369, + "nauc_precision_at_1_max": 0.48143439232866647, + "nauc_precision_at_1_std": 0.22002136550057733, + "nauc_precision_at_20_diff1": 0.3732459116186754, + "nauc_precision_at_20_max": 0.6731810217095847, + "nauc_precision_at_20_std": 0.4655216873912767, + "nauc_precision_at_3_diff1": 0.3911379992994151, + "nauc_precision_at_3_max": 0.5270269194433539, + "nauc_precision_at_3_std": 0.28440694428227226, + "nauc_precision_at_5_diff1": 0.37569190630366955, + "nauc_precision_at_5_max": 0.557310702722474, + "nauc_precision_at_5_std": 0.3082772997579572, + "nauc_recall_at_1000_diff1": 0.570746247216831, + "nauc_recall_at_1000_max": 0.9355383178912546, + "nauc_recall_at_1000_std": 0.8143719026071998, + "nauc_recall_at_100_diff1": 0.28561103871074306, + "nauc_recall_at_100_max": 0.6815610029184481, + "nauc_recall_at_100_std": 0.5854242763532237, + "nauc_recall_at_10_diff1": 0.34808730638537383, + "nauc_recall_at_10_max": 0.6230498863849571, + "nauc_recall_at_10_std": 0.38972169526152306, + "nauc_recall_at_1_diff1": 0.4897268439399369, + "nauc_recall_at_1_max": 0.48143439232866647, + "nauc_recall_at_1_std": 0.22002136550057733, + "nauc_recall_at_20_diff1": 0.3732459116186757, + "nauc_recall_at_20_max": 0.6731810217095842, + "nauc_recall_at_20_std": 0.46552168739127814, + "nauc_recall_at_3_diff1": 0.39113799929941523, + "nauc_recall_at_3_max": 0.5270269194433536, + "nauc_recall_at_3_std": 0.28440694428227187, + "nauc_recall_at_5_diff1": 0.3756919063036706, + "nauc_recall_at_5_max": 0.5573107027224745, + "nauc_recall_at_5_std": 0.3082772997579581, + "ndcg_at_1": 0.329, + "ndcg_at_10": 0.48558, + "ndcg_at_100": 0.53536, + "ndcg_at_1000": 0.5476, + "ndcg_at_20": 0.5087, + "ndcg_at_3": 0.42192, + "ndcg_at_5": 0.45426, + "precision_at_1": 0.329, + "precision_at_10": 0.06625, + "precision_at_100": 0.00898, + "precision_at_1000": 0.00099, + "precision_at_20": 0.03765, + "precision_at_3": 0.16267, + "precision_at_5": 0.1132, + "recall_at_1": 0.329, + "recall_at_10": 0.6625, + "recall_at_100": 0.898, + "recall_at_1000": 0.9935, + "recall_at_20": 0.753, + "recall_at_3": 0.488, + "recall_at_5": 0.566 + }, + { + "cv_recall_at_1": 0.7535, + "cv_recall_at_10": 0.9685, + "cv_recall_at_100": 0.9985, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.986, + "cv_recall_at_3": 0.8905, + "cv_recall_at_5": 0.939, + "hf_subset": "zh", + "languages": [ + "zho-Hans" + ], + "main_score": 0.8647, + "map_at_1": 0.7535, + "map_at_10": 0.83094, + "map_at_100": 0.83264, + "map_at_1000": 0.83265, + "map_at_20": 0.83224, + "map_at_3": 0.81583, + "map_at_5": 0.82686, + "mrr_at_1": 0.7535, + "mrr_at_10": 0.8309436507936507, + "mrr_at_100": 0.8326428461698397, + "mrr_at_1000": 0.8326528711689501, + "mrr_at_20": 0.8322375589172607, + "mrr_at_3": 0.815833333333333, + "mrr_at_5": 0.8268583333333337, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.9074074074074042, + "nauc_cv_recall_at_100_max": 0.9564270152506046, + "nauc_cv_recall_at_100_std": 0.08667911609083245, + "nauc_cv_recall_at_10_diff1": 0.7159604582573755, + "nauc_cv_recall_at_10_max": 0.7671216634802024, + "nauc_cv_recall_at_10_std": -0.3054925673973297, + "nauc_cv_recall_at_1_diff1": 0.810180475530634, + "nauc_cv_recall_at_1_max": 0.5523023359455304, + "nauc_cv_recall_at_1_std": -0.18382166913136913, + "nauc_cv_recall_at_20_diff1": 0.7574196345204801, + "nauc_cv_recall_at_20_max": 0.8418867547018848, + "nauc_cv_recall_at_20_std": -0.17697078831532298, + "nauc_cv_recall_at_3_diff1": 0.7287258127811141, + "nauc_cv_recall_at_3_max": 0.670665706437131, + "nauc_cv_recall_at_3_std": -0.1443513922609512, + "nauc_cv_recall_at_5_diff1": 0.718185088242946, + "nauc_cv_recall_at_5_max": 0.7166850346696072, + "nauc_cv_recall_at_5_std": -0.17488252131453574, + "nauc_map_at_1000_diff1": 0.7907323043547354, + "nauc_map_at_1000_max": 0.5861408086781905, + "nauc_map_at_1000_std": -0.1787536887590416, + "nauc_map_at_100_diff1": 0.7907384415401878, + "nauc_map_at_100_max": 0.5861665619023456, + "nauc_map_at_100_std": -0.17870425477150786, + "nauc_map_at_10_diff1": 0.7898007655152364, + "nauc_map_at_10_max": 0.58760606658666, + "nauc_map_at_10_std": -0.18104252909636068, + "nauc_map_at_1_diff1": 0.810180475530634, + "nauc_map_at_1_max": 0.5523023359455304, + "nauc_map_at_1_std": -0.18382166913136913, + "nauc_map_at_20_diff1": 0.7905669610329621, + "nauc_map_at_20_max": 0.586723738010071, + "nauc_map_at_20_std": -0.17897834402449417, + "nauc_map_at_3_diff1": 0.7848945769498309, + "nauc_map_at_3_max": 0.5899769317245516, + "nauc_map_at_3_std": -0.17212119267453208, + "nauc_map_at_5_diff1": 0.7879028117425809, + "nauc_map_at_5_max": 0.5892120425613406, + "nauc_map_at_5_std": -0.17707379971722223, + "nauc_mrr_at_1000_diff1": 0.7907323043547354, + "nauc_mrr_at_1000_max": 0.5861408086781905, + "nauc_mrr_at_1000_std": -0.1787536887590416, + "nauc_mrr_at_100_diff1": 0.7907384415401878, + "nauc_mrr_at_100_max": 0.5861665619023456, + "nauc_mrr_at_100_std": -0.17870425477150786, + "nauc_mrr_at_10_diff1": 0.7898007655152364, + "nauc_mrr_at_10_max": 0.58760606658666, + "nauc_mrr_at_10_std": -0.18104252909636068, + "nauc_mrr_at_1_diff1": 0.810180475530634, + "nauc_mrr_at_1_max": 0.5523023359455304, + "nauc_mrr_at_1_std": -0.18382166913136913, + "nauc_mrr_at_20_diff1": 0.7905669610329621, + "nauc_mrr_at_20_max": 0.586723738010071, + "nauc_mrr_at_20_std": -0.17897834402449417, + "nauc_mrr_at_3_diff1": 0.7848945769498309, + "nauc_mrr_at_3_max": 0.5899769317245516, + "nauc_mrr_at_3_std": -0.17212119267453208, + "nauc_mrr_at_5_diff1": 0.7879028117425809, + "nauc_mrr_at_5_max": 0.5892120425613406, + "nauc_mrr_at_5_std": -0.17707379971722223, + "nauc_ndcg_at_1000_diff1": 0.7886545545175407, + "nauc_ndcg_at_1000_max": 0.5912553799694619, + "nauc_ndcg_at_1000_std": -0.17878692861317544, + "nauc_ndcg_at_100_diff1": 0.7888518971892833, + "nauc_ndcg_at_100_max": 0.5919000390709643, + "nauc_ndcg_at_100_std": -0.1781301914289305, + "nauc_ndcg_at_10_diff1": 0.7841553474017766, + "nauc_ndcg_at_10_max": 0.6004233460965622, + "nauc_ndcg_at_10_std": -0.18770318177002843, + "nauc_ndcg_at_1_diff1": 0.810180475530634, + "nauc_ndcg_at_1_max": 0.5523023359455304, + "nauc_ndcg_at_1_std": -0.18382166913136913, + "nauc_ndcg_at_20_diff1": 0.7877499835627467, + "nauc_ndcg_at_20_max": 0.5966008749588094, + "nauc_ndcg_at_20_std": -0.17918935976242298, + "nauc_ndcg_at_3_diff1": 0.7746259970314816, + "nauc_ndcg_at_3_max": 0.6047740676129029, + "nauc_ndcg_at_3_std": -0.1670716308750744, + "nauc_ndcg_at_5_diff1": 0.779426755716452, + "nauc_ndcg_at_5_max": 0.6045320925860991, + "nauc_ndcg_at_5_std": -0.17646919507329487, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.9074074074073368, + "nauc_precision_at_100_max": 0.9564270152505274, + "nauc_precision_at_100_std": 0.0866791160907527, + "nauc_precision_at_10_diff1": 0.7159604582573762, + "nauc_precision_at_10_max": 0.7671216634802053, + "nauc_precision_at_10_std": -0.30549256739733754, + "nauc_precision_at_1_diff1": 0.810180475530634, + "nauc_precision_at_1_max": 0.5523023359455304, + "nauc_precision_at_1_std": -0.18382166913136913, + "nauc_precision_at_20_diff1": 0.7574196345204598, + "nauc_precision_at_20_max": 0.8418867547018706, + "nauc_precision_at_20_std": -0.1769707883153585, + "nauc_precision_at_3_diff1": 0.7287258127811156, + "nauc_precision_at_3_max": 0.6706657064371305, + "nauc_precision_at_3_std": -0.14435139226094928, + "nauc_precision_at_5_diff1": 0.7181850882429455, + "nauc_precision_at_5_max": 0.7166850346696044, + "nauc_precision_at_5_std": -0.17488252131453935, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.9074074074074042, + "nauc_recall_at_100_max": 0.9564270152506046, + "nauc_recall_at_100_std": 0.08667911609083245, + "nauc_recall_at_10_diff1": 0.7159604582573755, + "nauc_recall_at_10_max": 0.7671216634802024, + "nauc_recall_at_10_std": -0.3054925673973297, + "nauc_recall_at_1_diff1": 0.810180475530634, + "nauc_recall_at_1_max": 0.5523023359455304, + "nauc_recall_at_1_std": -0.18382166913136913, + "nauc_recall_at_20_diff1": 0.7574196345204801, + "nauc_recall_at_20_max": 0.8418867547018848, + "nauc_recall_at_20_std": -0.17697078831532298, + "nauc_recall_at_3_diff1": 0.7287258127811141, + "nauc_recall_at_3_max": 0.670665706437131, + "nauc_recall_at_3_std": -0.1443513922609512, + "nauc_recall_at_5_diff1": 0.718185088242946, + "nauc_recall_at_5_max": 0.7166850346696072, + "nauc_recall_at_5_std": -0.17488252131453574, + "ndcg_at_1": 0.7535, + "ndcg_at_10": 0.8647, + "ndcg_at_100": 0.87171, + "ndcg_at_1000": 0.87191, + "ndcg_at_20": 0.86922, + "ndcg_at_3": 0.83509, + "ndcg_at_5": 0.85502, + "precision_at_1": 0.7535, + "precision_at_10": 0.09685, + "precision_at_100": 0.00999, + "precision_at_1000": 0.001, + "precision_at_20": 0.0493, + "precision_at_3": 0.29683, + "precision_at_5": 0.1878, + "recall_at_1": 0.7535, + "recall_at_10": 0.9685, + "recall_at_100": 0.9985, + "recall_at_1000": 1.0, + "recall_at_20": 0.986, + "recall_at_3": 0.8905, + "recall_at_5": 0.939 + } + ] + }, + "task_name": "XFlickr30kCoT2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/voyage-multimodal-3/1/model_meta.json b/results-mieb/voyage-multimodal-3/1/model_meta.json new file mode 100644 index 0000000000..528c812ae0 --- /dev/null +++ b/results-mieb/voyage-multimodal-3/1/model_meta.json @@ -0,0 +1 @@ +{"name": "voyage-multimodal-3", "revision": "1", "release_date": "2024-11-10", "languages": [], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": 1024, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": "cosine", "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "voyage_v_loader"} \ No newline at end of file